feat(FN-4813): complete docs, settings, and verification updates

Fusion-Task-Id: FN-4813
Fusion-Task-Lineage: 846893a5-2afa-4817-8f64-8c444d2fd713

.changeset/FN-4813-multi-node-claim-mutex.md

@@ -0,0 +1,5 @@
+---
+"@runfusion/fusion": patch
+---
+
+Add multi-node coordination hardening for task execution: distributed checkout claim mutex (`tryClaimCheckout`) with node/epoch preconditions, configurable `owningNodeHandoffPolicy` behavior for unavailable owners, and a supported `transitionProjectIsolation` path that can restart project runtimes (with rollback when active-task restart is blocked). Reaffirm scheduler failover and live process migration as explicit non-goals in mesh/multi-project docs.

docs/settings-reference.md

@@ -86,6 +86,7 @@ Fusion automatically falls back to ntfy's JSON publish format when a notificatio
 | `settingsSyncAuth` | `boolean` | `false` | Include auth-material snapshots (`sharedState.authMaterial` and auth sync endpoints) when settings sync is enabled. Ignored when `settingsSyncEnabled` is `false`. |
 | `settingsSyncInterval` | `number` | `900000` | Automatic sync interval in ms. Valid values: `300000`, `900000`, `1800000`, `3600000`. |
 | `settingsSyncConflictResolution` | `"last-write-wins" \| "always-ask" \| "keep-local" \| "keep-remote"` | `"last-write-wins"` | Conflict strategy for divergent synced settings. |
+| `owningNodeHandoffPolicy` | `"block" \| "reassign-to-local" \| "reassign-any-healthy"` | `"reassign-to-local"` | Global fallback policy for tasks whose owning checkout node is unavailable. Project-level `owningNodeHandoffPolicy` overrides this. |
 | `dashboardCurrentNodeId` | `string` | `undefined` | Currently selected dashboard node ID. Restores the last-viewed node on fresh browser/PWA sessions. `undefined` means viewing the local node. |
 
 > Mesh lifecycle note: settings sync is executed by the process-level `PeerExchangeService` started by `fn serve`/`fn dashboard`. `InProcessRuntime` does not instantiate settings-sync mesh services per project.
@@ -189,6 +190,7 @@ Defaults from `DEFAULT_PROJECT_SETTINGS`; key scope from `PROJECT_SETTINGS_KEYS`
 | `autoClaimCandidatesInPrompt` | `number` | `5` | Default no-task heartbeat candidate list length. Integer range `0-10`; `0` suppresses candidate prompt injection. |
 | `defaultNodeId` | `string` | `undefined` | Optional project default execution node for task dispatch. When set, tasks without a per-task `nodeId` override resolve to this node (`routing source: project-default`). See [Task Management → Node Routing](./task-management.md#node-routing). |
 | `unavailableNodePolicy` | `"block" \| "fallback-local"` | `"block"` | Project routing policy used during scheduler dispatch when a task resolves to a remote node and node health is known. `"block"` keeps the task in `todo` if the node is unhealthy; `"fallback-local"` reroutes dispatch to local execution. See [Architecture → Task Routing Architecture](./architecture.md#task-routing-architecture). |
+| `owningNodeHandoffPolicy` | `"block" \| "reassign-to-local" \| "reassign-any-healthy"` | `"reassign-to-local"` | Policy for tasks already checked out by an unavailable owning node. `"block"` parks, `"reassign-to-local"` takes over on local node, `"reassign-any-healthy"` makes takeover eligible on healthy peers. |
 
 | `groupOverlappingFiles` | `boolean` | `true` | Serialize execution when file scopes overlap. |
 | `pluginTrustPolicy` | `"off" | "warn" | "enforce"` | `"warn"` | Plugin provenance enforcement mode: `off` records verification metadata only, `warn` blocks only `invalid` signatures, `enforce` allows only `verified-trusted` or `trusted-local`. |
@@ -516,6 +518,7 @@ Node routing controls in the project settings table are configured from **Settin
 
 - `fn settings set defaultNodeId <node-id>`
 - `fn settings set unavailableNodePolicy <block|fallback-local>`
+- `fn settings set owningNodeHandoffPolicy <block|reassign-to-local|reassign-any-healthy>`
 
 Routing precedence for task dispatch is:
 1. per-task override (`Task.nodeId`)

packages/core/src/__tests__/agent-store.test.ts

@@ -1848,14 +1848,14 @@ describe("AgentStore", () => {
     });
 
     it("checkoutTask acquires a lease and stamps lease metadata", async () => {
-      const updated = await store.checkoutTask(holderId, taskId, { nodeId: "node-a", runId: "run-1", leaseEpoch: 2 });
+      const updated = await store.checkoutTask(holderId, taskId, { nodeId: "node-a", runId: "run-1", leaseEpoch: 0 });
 
       expect(updated.checkedOutBy).toBe(holderId);
       expect(updated.checkedOutAt).toBeDefined();
       expect(updated.checkoutNodeId).toBe("node-a");
       expect(updated.checkoutRunId).toBe("run-1");
       expect(updated.checkoutLeaseRenewedAt).toBeDefined();
-      expect(updated.checkoutLeaseEpoch).toBe(2);
+      expect(updated.checkoutLeaseEpoch).toBeGreaterThanOrEqual(1);
 
       const persisted = await taskStore.getTask(taskId);
       expect(persisted?.checkedOutBy).toBe(holderId);
@@ -1863,29 +1863,31 @@ describe("AgentStore", () => {
       expect(persisted?.checkoutNodeId).toBe("node-a");
       expect(persisted?.checkoutRunId).toBe("run-1");
       expect(persisted?.checkoutLeaseRenewedAt).toBeDefined();
-      expect(persisted?.checkoutLeaseEpoch).toBe(2);
+      expect(persisted?.checkoutLeaseEpoch).toBe(updated.checkoutLeaseEpoch);
     });
 
     it("checkoutTask is idempotent for same agent/node/epoch and renews lease timestamp", async () => {
-      const first = await store.checkoutTask(holderId, taskId, { nodeId: "node-a", runId: "run-1", leaseEpoch: 2 });
+      const first = await store.checkoutTask(holderId, taskId, { nodeId: "node-a", runId: "run-1", leaseEpoch: 0 });
       await new Promise((resolve) => setTimeout(resolve, 5));
-      const second = await store.checkoutTask(holderId, taskId, { nodeId: "node-a", runId: "run-2", leaseEpoch: 2 });
+      const second = await store.checkoutTask(holderId, taskId, {
+        nodeId: "node-a",
+        runId: "run-2",
+        leaseEpoch: first.checkoutLeaseEpoch ?? 0,
+      });
 
       expect(second.checkedOutBy).toBe(holderId);
       expect(second.checkedOutAt).toBe(first.checkedOutAt);
       expect(second.checkoutNodeId).toBe("node-a");
       expect(second.checkoutRunId).toBe("run-2");
-      expect(second.checkoutLeaseEpoch).toBe(2);
+      expect(second.checkoutLeaseEpoch).toBe(first.checkoutLeaseEpoch);
       expect(second.checkoutLeaseRenewedAt).not.toBe(first.checkoutLeaseRenewedAt);
     });
 
-    it("checkoutTask updates epoch for same holder when lease epoch increases", async () => {
-      await store.checkoutTask(holderId, taskId, { nodeId: "node-a", runId: "run-1", leaseEpoch: 1 });
-      const bumped = await store.checkoutTask(holderId, taskId, { nodeId: "node-a", runId: "run-2", leaseEpoch: 3 });
-
-      expect(bumped.checkedOutBy).toBe(holderId);
-      expect(bumped.checkoutLeaseEpoch).toBe(3);
-      expect(bumped.checkoutRunId).toBe("run-2");
+    it("checkoutTask rejects renewal attempts with a mismatched epoch", async () => {
+      await store.checkoutTask(holderId, taskId, { nodeId: "node-a", runId: "run-1", leaseEpoch: 0 });
+      await expect(
+        store.checkoutTask(holderId, taskId, { nodeId: "node-a", runId: "run-2", leaseEpoch: 3 }),
+      ).rejects.toBeInstanceOf(CheckoutConflictError);
     });
 
     it("checkoutTask throws CheckoutConflictError when already held by another agent", async () => {

packages/core/src/__tests__/settings-parity.test.ts

@@ -226,7 +226,12 @@ describe("settings key parity", () => {
   it("only intentional shared keys appear in both global and project scopes", () => {
     const projectKeySet = new Set(PROJECT_SETTINGS_KEYS as readonly string[]);
     const overlap = (GLOBAL_SETTINGS_KEYS as readonly string[]).filter((key) => projectKeySet.has(key));
-    expect(overlap).toEqual(["taskTokenBudget", "githubTrackingDefaultRepo", "worktrunk"]);
+    expect(overlap).toEqual([
+      "taskTokenBudget",
+      "githubTrackingDefaultRepo",
+      "worktrunk",
+      "owningNodeHandoffPolicy",
+    ]);
   });
 });
 

packages/core/src/types.ts

@@ -846,7 +846,15 @@ export interface TaskLogEntry {
   runContext?: RunMutationContext;
 }
 
-export type ActivityEventType = "task:created" | "task:moved" | "task:updated" | "task:deleted" | "task:merged" | "task:failed" | "settings:updated";
+export type ActivityEventType =
+  | "task:created"
+  | "task:moved"
+  | "task:updated"
+  | "task:deleted"
+  | "task:merged"
+  | "task:failed"
+  | "settings:updated"
+  | "project:isolation-transition";
 
 export interface ActivityLogEntry {
   id: string;

packages/engine/src/__tests__/owning-node-handoff-policy.test.ts

@@ -10,7 +10,6 @@ const baseTask: Task = {
   steps: [],
   currentStep: 0,
   log: [],
-  prompt: "",
   createdAt: "2026-01-01T00:00:00.000Z",
   updatedAt: "2026-01-01T00:00:00.000Z",
 };

packages/engine/src/__tests__/owning-node-handoff.integration.test.ts

@@ -32,7 +32,6 @@ describe("MeshLeaseManager owning-node handoff integration", () => {
 
   async function seedLease(ownerNodeId: string): Promise<void> {
     await taskStore.updateTask(taskId, {
-      column: "in-progress",
       checkedOutBy: "agent-1",
       checkedOutAt: "2026-05-01T00:00:00.000Z",
       checkoutLeaseRenewedAt: "2026-05-01T00:00:00.000Z",

packages/engine/src/scheduler.ts

@@ -1029,7 +1029,9 @@ export class Scheduler {
             }
           }
 
-          const nodeHealth = this.options.nodeHealthMonitor.getNodeHealth(effectiveNode.nodeId);
+          const nodeHealth = effectiveNode.nodeId
+            ? this.options.nodeHealthMonitor.getNodeHealth(effectiveNode.nodeId)
+            : undefined;
           const decision = applyUnavailableNodePolicy({
             effectiveNode,
             nodeHealth,