feat(FN-4813): complete Step 5 — add isolation transition runtime restart path

Fusion-Task-Id: FN-4813
Fusion-Task-Lineage: 846893a5-2afa-4817-8f64-8c444d2fd713
This commit is contained in:
Fusion (runfusion.ai)
2026-05-16 17:03:12 -07:00
committed by gsxdsm
parent 2ca18e983a
commit 63adddadf9
10 changed files with 513 additions and 96 deletions

View File

@@ -0,0 +1,53 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { mkdirSync, mkdtempSync } from "node:fs";
import { rm } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { CentralCore } from "../central-core.js";
describe("CentralCore.transitionProjectIsolation", () => {
let tempDir: string;
let projectPath: string;
let core: CentralCore;
beforeEach(async () => {
tempDir = mkdtempSync(join(tmpdir(), "fn-project-isolation-transition-"));
projectPath = join(tempDir, "project");
mkdirSync(projectPath, { recursive: true });
core = new CentralCore(tempDir);
await core.init();
});
afterEach(async () => {
await core.close();
await rm(tempDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
});
it("returns noop when next mode matches existing mode", async () => {
const project = await core.registerProject({
name: "Test",
path: projectPath,
isolationMode: "in-process",
});
const result = await core.transitionProjectIsolation(project.id, "in-process");
expect(result).toEqual({ ok: false, reason: "noop" });
});
it("updates mode and logs activity on success", async () => {
const project = await core.registerProject({
name: "Test",
path: projectPath,
isolationMode: "in-process",
});
const result = await core.transitionProjectIsolation(project.id, "child-process");
expect(result).toEqual({ ok: true });
const updated = await core.getProject(project.id);
expect(updated?.isolationMode).toBe("child-process");
const activity = await core.getRecentActivity({ projectId: project.id, limit: 10 });
expect(activity.some((entry) => entry.type === "project:isolation-transition")).toBe(true);
});
});

View File

@@ -514,6 +514,40 @@ export class CentralCore extends EventEmitter<CentralCoreEvents> {
return updated;
}
async transitionProjectIsolation(
projectId: string,
nextMode: IsolationMode,
opts: { force?: boolean } = {}
): Promise<{ ok: true } | { ok: false; reason: string; activeTaskCount?: number }> {
void opts;
this.ensureInitialized();
const project = await this.getProject(projectId);
if (!project) {
throw new Error(`Project not found: ${projectId}`);
}
if (project.isolationMode === nextMode) {
return { ok: false, reason: "noop" };
}
await this.updateProject(projectId, { isolationMode: nextMode });
await this.logActivity({
type: "project:isolation-transition",
projectId,
projectName: project.name,
timestamp: new Date().toISOString(),
details: `Project isolation mode transitioned: ${project.isolationMode} -> ${nextMode}`,
metadata: {
from: project.isolationMode,
to: nextMode,
},
});
return { ok: true };
}
/**
* Reconcile stale project statuses.
*

View File

@@ -0,0 +1,108 @@
// @vitest-environment node
import { beforeEach, describe, expect, it, vi } from "vitest";
import express from "express";
import { createApiRoutes } from "../routes.js";
import { request } from "../test-request.js";
import type { TaskStore } from "@fusion/core";
const project = {
id: "proj_1",
name: "Project",
path: "/tmp/project",
status: "active",
isolationMode: "in-process" as const,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
};
const central = {
isInitialized: () => true,
getProject: vi.fn(),
updateProject: vi.fn(),
unassignProjectFromNode: vi.fn(),
assignProjectToNode: vi.fn(),
};
function createStore(): TaskStore {
return {
listTasks: vi.fn().mockResolvedValue([]),
getSettings: vi.fn().mockResolvedValue({}),
getSettingsFast: vi.fn().mockResolvedValue({}),
getRootDir: vi.fn().mockReturnValue("/tmp/test"),
getFusionDir: vi.fn().mockReturnValue("/tmp/test/.fusion"),
getPluginStore: vi.fn().mockReturnValue({ listPlugins: vi.fn().mockResolvedValue([]) }),
getMissionStore: vi.fn().mockReturnValue({ listMissions: vi.fn().mockResolvedValue([]) }),
getRoutineStore: vi.fn().mockReturnValue({ listRoutines: vi.fn().mockResolvedValue([]) }),
getAutomationStore: vi.fn().mockReturnValue({ listScheduledTasks: vi.fn().mockResolvedValue([]) }),
} as unknown as TaskStore;
}
function createApp(options?: Parameters<typeof createApiRoutes>[1]) {
const app = express();
app.use(express.json());
app.use("/api", createApiRoutes(createStore(), { ...options, centralCore: central as any }));
return app;
}
describe("project isolation transition route", () => {
beforeEach(() => {
vi.clearAllMocks();
central.getProject.mockResolvedValue(project);
central.updateProject.mockImplementation(async (_id: string, updates: Record<string, unknown>) => ({ ...project, ...updates }));
});
it("uses hybrid executor transition path when available", async () => {
const hybridExecutor = { transitionProjectIsolation: vi.fn().mockResolvedValue({ ok: true }) };
const res = await request(
createApp({ hybridExecutor: hybridExecutor as any }),
"PATCH",
"/api/projects/proj_1",
JSON.stringify({ isolationMode: "child-process" }),
{ "content-type": "application/json" },
);
expect(res.status).toBe(200);
expect(hybridExecutor.transitionProjectIsolation).toHaveBeenCalledWith("proj_1", "child-process", { force: false });
expect((res.body as any).transitionDeferred).toBeUndefined();
});
it("falls back to direct update and marks transitionDeferred when no hybrid executor", async () => {
const res = await request(
createApp(),
"PATCH",
"/api/projects/proj_1",
JSON.stringify({ isolationMode: "child-process" }),
{ "content-type": "application/json" },
);
expect(res.status).toBe(200);
expect((res.body as any).transitionDeferred).toBe(true);
});
it("returns 409 on active_tasks without force and succeeds with force", async () => {
const hybridExecutor = {
transitionProjectIsolation: vi
.fn()
.mockResolvedValueOnce({ ok: false, reason: "active_tasks", activeTaskCount: 2 })
.mockResolvedValueOnce({ ok: true }),
};
const blocked = await request(
createApp({ hybridExecutor: hybridExecutor as any }),
"PATCH",
"/api/projects/proj_1",
JSON.stringify({ isolationMode: "child-process" }),
{ "content-type": "application/json" },
);
expect(blocked.status).toBe(409);
const forced = await request(
createApp({ hybridExecutor: hybridExecutor as any }),
"PATCH",
"/api/projects/proj_1",
JSON.stringify({ isolationMode: "child-process", force: true }),
{ "content-type": "application/json" },
);
expect(forced.status).toBe(200);
expect(hybridExecutor.transitionProjectIsolation).toHaveBeenLastCalledWith("proj_1", "child-process", { force: true });
});
});

View File

@@ -585,33 +585,63 @@ export const registerProjectRoutes: ApiRouteRegistrar = (ctx) => {
*/
router.patch("/projects/:id", async (req, res) => {
try {
const { name, status, isolationMode, nodeId } = req.body;
const { name, status, isolationMode, nodeId, force } = req.body;
const updates: Partial<import("@fusion/core").RegisteredProject> = {};
if (name !== undefined) updates.name = name;
if (status !== undefined) updates.status = status as import("@fusion/core").ProjectStatus;
if (isolationMode !== undefined) updates.isolationMode = isolationMode as "in-process" | "child-process";
const resultProject = await withCentralCore(async (central) => {
const project = await central.updateProject(req.params.id, updates);
if (!project) {
const result = await withCentralCore(async (central) => {
const existing = await central.getProject(req.params.id);
if (!existing) {
throw notFound("Project not found");
}
let transitionDeferred = false;
const isolationChanged =
isolationMode !== undefined && isolationMode !== existing.isolationMode;
if (isolationChanged) {
if (options?.hybridExecutor) {
const transition = await options.hybridExecutor.transitionProjectIsolation(
req.params.id,
isolationMode as "in-process" | "child-process",
{ force: Boolean(force) },
);
if (!transition.ok && transition.reason === "active_tasks") {
throw new ApiError(409, "active_tasks", {
error: "active_tasks",
activeTaskCount: transition.activeTaskCount ?? 0,
});
}
} else {
transitionDeferred = true;
}
}
const project = await central.updateProject(req.params.id, updates);
if (nodeId === undefined) {
return project;
return { project, transitionDeferred };
}
if (nodeId === null) {
return await central.unassignProjectFromNode(req.params.id);
return {
project: await central.unassignProjectFromNode(req.params.id),
transitionDeferred,
};
}
if (typeof nodeId === "string" && nodeId.trim()) {
return await central.assignProjectToNode(req.params.id, nodeId.trim());
return {
project: await central.assignProjectToNode(req.params.id, nodeId.trim()),
transitionDeferred,
};
}
throw badRequest("nodeId must be a non-empty string or null");
});
res.json(resultProject);
res.json(result.transitionDeferred ? { ...result.project, transitionDeferred: true } : result.project);
} catch (err: unknown) {
if (err instanceof ApiError) {
throw err;

View File

@@ -16,6 +16,7 @@ const mockProjectManagerInstances: Array<{
getGlobalMetrics: ReturnType<typeof vi.fn>;
acquireGlobalSlot: ReturnType<typeof vi.fn>;
releaseGlobalSlot: ReturnType<typeof vi.fn>;
restartProjectRuntime: ReturnType<typeof vi.fn>;
stopAll: ReturnType<typeof vi.fn>;
on: ReturnType<typeof vi.fn>;
}> = [];
@@ -71,6 +72,7 @@ vi.mock("../project-manager.js", () => ({
}),
acquireGlobalSlot: vi.fn().mockResolvedValue(true),
releaseGlobalSlot: vi.fn().mockResolvedValue(undefined),
restartProjectRuntime: vi.fn().mockResolvedValue(undefined),
stopAll: vi.fn().mockImplementation(() => {
mockRuntimes.clear();
mockProjectIds.length = 0;
@@ -133,6 +135,7 @@ describe("HybridExecutor", () => {
registerProject: vi.fn().mockResolvedValue(mockProject),
unregisterProject: vi.fn().mockResolvedValue(undefined),
updateProject: vi.fn().mockResolvedValue(mockProject),
transitionProjectIsolation: vi.fn().mockResolvedValue({ ok: true }),
getGlobalConcurrencyState: vi.fn().mockResolvedValue({
globalMaxConcurrent: 4,
currentlyActive: 0,
@@ -475,4 +478,26 @@ describe("HybridExecutor", () => {
);
});
});
describe("transitionProjectIsolation", () => {
beforeEach(async () => {
await executor.initialize();
await executor.addProject({
projectId: "proj_test123",
workingDirectory: "/tmp/test-project",
isolationMode: "in-process",
maxConcurrent: 2,
maxWorktrees: 4,
});
});
it("rolls back persisted isolation mode when restart is blocked by active tasks", async () => {
const manager = mockProjectManagerInstances[0];
manager?.restartProjectRuntime.mockRejectedValueOnce({ kind: "active_tasks", count: 3 });
const result = await executor.transitionProjectIsolation("proj_test123", "child-process");
expect(result).toEqual({ ok: false, reason: "active_tasks", activeTaskCount: 3 });
expect(mockCentralCore.updateProject).toHaveBeenCalledWith("proj_test123", { isolationMode: "in-process" });
});
});
});

View File

@@ -1,71 +1,82 @@
import { describe, expect, it, vi } from "vitest";
import type { Task, TaskStore } from "@fusion/core";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { mkdtempSync } from "node:fs";
import { rm } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { TaskStore, type OwningNodeHandoffPolicy } from "@fusion/core";
import { MeshLeaseManager } from "../mesh-lease-manager.js";
import type { NodeHealthMonitor } from "../node-health-monitor.js";
function baseTask(overrides: Partial<Task> = {}): Task {
return {
id: "FN-1",
description: "handoff",
column: "in-progress",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: "2026-01-01T00:00:00.000Z",
updatedAt: "2026-01-01T00:00:00.000Z",
checkedOutBy: "agent-1",
checkedOutAt: "2026-01-01T00:00:00.000Z",
checkoutLeaseRenewedAt: "2026-01-01T00:00:00.000Z",
checkoutLeaseEpoch: 1,
checkoutNodeId: "node-owner",
...overrides,
};
}
function createTaskStore(task: Task) {
let current = { ...task };
const updateTask = vi.fn(async (_id: string, patch: Partial<Task>) => {
current = { ...current, ...patch };
return current;
});
const taskStore = {
getTask: vi.fn(async () => current),
updateTask,
moveTask: vi.fn(async () => current),
logEntry: vi.fn(async () => undefined),
} as unknown as TaskStore;
return { taskStore, updateTask, getCurrent: () => current };
function makeTmpDir(): string {
return mkdtempSync(join(tmpdir(), "fn-owning-handoff-test-"));
}
describe("MeshLeaseManager owning-node handoff integration", () => {
it.each([
{ policy: "block", selfOwned: false, expectRecovered: false },
{ policy: "reassign-to-local", selfOwned: false, expectRecovered: true },
{ policy: "reassign-any-healthy", selfOwned: false, expectRecovered: true },
{ policy: "block", selfOwned: true, expectRecovered: true },
{ policy: "reassign-to-local", selfOwned: true, expectRecovered: true },
{ policy: "reassign-any-healthy", selfOwned: true, expectRecovered: true },
] as const)("policy=$policy selfOwned=$selfOwned", async ({ policy, selfOwned, expectRecovered }) => {
const ownerNodeId = selfOwned ? "node-local" : "node-owner";
const { taskStore, updateTask, getCurrent } = createTaskStore(baseTask({ checkoutNodeId: ownerNodeId }));
let rootDir: string;
let globalDir: string;
let taskStore: TaskStore;
let taskId: string;
beforeEach(async () => {
rootDir = makeTmpDir();
globalDir = join(rootDir, ".fusion-global");
taskStore = new TaskStore(rootDir, globalDir);
await taskStore.init();
taskId = (await taskStore.createTask({ description: "handoff" })).id;
});
afterEach(async () => {
taskStore?.close();
await rm(rootDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
});
async function seedLease(ownerNodeId: string): Promise<void> {
await taskStore.updateTask(taskId, {
column: "in-progress",
checkedOutBy: "agent-1",
checkedOutAt: "2026-05-01T00:00:00.000Z",
checkoutLeaseRenewedAt: "2026-05-01T00:00:00.000Z",
checkoutLeaseEpoch: 1,
checkoutNodeId: ownerNodeId,
});
}
async function runCase(policy: OwningNodeHandoffPolicy, ownerNodeId: string): Promise<boolean> {
const manager = new MeshLeaseManager({
taskStore,
nodeHealthMonitor: { getNodeHealth: vi.fn(() => "offline") } as any,
localNodeId: "node-local",
getHandoffPolicy: async () => policy,
nodeHealthMonitor: {
getNodeHealth: () => "offline",
} as unknown as NodeHealthMonitor,
});
return manager.recoverAbandonedLease(taskId, "test-owner-unavailable", { preserveProgress: true });
}
const recovered = await manager.recoverAbandonedLease("FN-1", "test");
expect(recovered).toBe(expectRecovered);
it("applies handoff policy matrix for peer-owned leases", async () => {
await seedLease("node-peer");
expect(await runCase("block", "node-peer")).toBe(false);
let task = await taskStore.getTask(taskId);
expect(task?.checkedOutBy).toBe("agent-1");
if (expectRecovered) {
expect(updateTask).toHaveBeenCalled();
expect(getCurrent().checkedOutBy).toBeNull();
expect(getCurrent().checkoutNodeId).toBeNull();
} else {
expect(updateTask).not.toHaveBeenCalled();
expect(getCurrent().checkedOutBy).toBe("agent-1");
await seedLease("node-peer");
expect(await runCase("reassign-to-local", "node-peer")).toBe(true);
task = await taskStore.getTask(taskId);
expect(task?.checkedOutBy ?? null).toBeNull();
await seedLease("node-peer");
expect(await runCase("reassign-any-healthy", "node-peer")).toBe(true);
task = await taskStore.getTask(taskId);
expect(task?.checkedOutBy ?? null).toBeNull();
});
it("recovers self-owned leases regardless of policy", async () => {
for (const policy of ["block", "reassign-to-local", "reassign-any-healthy"] as const) {
await seedLease("node-local");
const recovered = await runCase(policy, "node-local");
expect(recovered).toBe(true);
const task = await taskStore.getTask(taskId);
expect(task?.checkedOutBy ?? null).toBeNull();
}
});
});

View File

@@ -401,4 +401,67 @@ describe("Scheduler node routing", () => {
expect((healthMonitor.getNodeHealth as ReturnType<typeof vi.fn>)).not.toHaveBeenCalled();
});
it("parks dispatch when owning-node handoff policy blocks peer-owner takeover", async () => {
const task = createMockTask({ id: "FN-118", nodeId: "node-online", checkoutNodeId: "node-owner", checkedOutBy: "agent-owner" });
const store = createMockStore(task, {
maxConcurrent: 1,
maxWorktrees: 1,
unavailableNodePolicy: "block",
owningNodeHandoffPolicy: "block",
});
const healthMonitor = createMockHealthMonitor({ "node-online": "online", "node-owner": "offline" });
const scheduler = new Scheduler(store, { nodeHealthMonitor: healthMonitor });
(scheduler as unknown as { running: boolean }).running = true;
await scheduler.schedule();
expect(store.updateTask).not.toHaveBeenCalled();
expect(store.logEntry).toHaveBeenCalledWith(
task.id,
"Owning-node handoff parked dispatch: handoff_blocked_by_policy",
);
});
it("forces local dispatch when owning-node handoff returns reassign-local", async () => {
const task = createMockTask({ id: "FN-119", nodeId: "node-online", checkoutNodeId: "node-owner", checkedOutBy: "agent-owner" });
const store = createMockStore(task, {
maxConcurrent: 1,
maxWorktrees: 1,
unavailableNodePolicy: "block",
owningNodeHandoffPolicy: "reassign-to-local",
});
const healthMonitor = createMockHealthMonitor({ "node-online": "online", "node-owner": "offline" });
const scheduler = new Scheduler(store, { nodeHealthMonitor: healthMonitor });
(scheduler as unknown as { running: boolean }).running = true;
await scheduler.schedule();
expect(store.updateTask).toHaveBeenCalledWith(task.id, expect.objectContaining({
effectiveNodeId: null,
effectiveNodeSource: "local",
}));
expect(store.logEntry).toHaveBeenCalledWith(task.id, "Owning-node handoff applied: owner_offline_local_takes_over");
});
it("keeps non-local routing when owning-node handoff returns reassign-any", async () => {
const task = createMockTask({ id: "FN-120", nodeId: "node-online", checkoutNodeId: "node-owner", checkedOutBy: "agent-owner" });
const store = createMockStore(task, {
maxConcurrent: 1,
maxWorktrees: 1,
unavailableNodePolicy: "block",
owningNodeHandoffPolicy: "reassign-any-healthy",
});
const healthMonitor = createMockHealthMonitor({ "node-online": "online", "node-owner": "offline" });
const scheduler = new Scheduler(store, { nodeHealthMonitor: healthMonitor });
(scheduler as unknown as { running: boolean }).running = true;
await scheduler.schedule();
expect(store.updateTask).toHaveBeenCalledWith(task.id, expect.objectContaining({
effectiveNodeId: "node-online",
effectiveNodeSource: "task-override",
}));
expect(store.logEntry).toHaveBeenCalledWith(task.id, "Owning-node handoff applied: owner_offline_any_healthy_eligible");
});
});

View File

@@ -1,5 +1,5 @@
import { EventEmitter } from "node:events";
import type { Task, CentralCore, RegisteredProject } from "@fusion/core";
import type { Task, CentralCore, RegisteredProject, IsolationMode } from "@fusion/core";
import { ProjectManager } from "./project-manager.js";
import { NodeHealthMonitor } from "./node-health-monitor.js";
import type {
@@ -48,6 +48,8 @@ export interface HybridExecutorEvents {
"project:added": [data: { projectId: string; projectName: string }];
/** Emitted when a project runtime is removed */
"project:removed": [data: { projectId: string; projectName: string }];
/** Emitted when a project runtime is restarted */
"project:runtime-restarted": [data: { projectId: string; projectName: string; isolationMode: IsolationMode; reason?: string }];
}
/**
@@ -309,6 +311,47 @@ export class HybridExecutor extends EventEmitter<HybridExecutorEvents> {
return existingRuntime;
}
/**
* Transition project isolation mode and restart runtime to apply changes.
*
* Non-force transitions that fail runtime restart due to active tasks roll back
* the persisted isolationMode change to its previous value before returning.
*/
async transitionProjectIsolation(
projectId: string,
nextMode: IsolationMode,
opts?: { force?: boolean },
): Promise<{ ok: true } | { ok: false; reason: string; activeTaskCount?: number }> {
const current = await this.centralCore.getProject(projectId);
if (!current) {
return { ok: false, reason: "project_not_found" };
}
const transition = await this.centralCore.transitionProjectIsolation(projectId, nextMode, opts);
if (!transition.ok) {
return transition;
}
try {
await this.projectManager.restartProjectRuntime(projectId, {
reason: `isolation-transition:${current.isolationMode}->${nextMode}`,
force: opts?.force,
});
return { ok: true };
} catch (error) {
if (!opts?.force && error && typeof error === "object" && (error as { kind?: unknown }).kind === "active_tasks") {
await this.centralCore.updateProject(projectId, { isolationMode: current.isolationMode });
return {
ok: false,
reason: "active_tasks",
activeTaskCount: Number((error as { count?: unknown }).count) || 0,
};
}
throw error;
}
}
/**
* Get a runtime by project ID.
*/
@@ -438,6 +481,10 @@ export class HybridExecutor extends EventEmitter<HybridExecutorEvents> {
this.projectManager.on("runtime:removed", (data) => {
this.emit("project:removed", data);
});
this.projectManager.on("project:runtime-restarted", (data) => {
this.emit("project:runtime-restarted", data);
});
}
/**

View File

@@ -45,6 +45,21 @@ export interface ProjectManagerEvents {
"runtime:added": [data: { projectId: string; projectName: string }];
/** Emitted when a runtime is removed */
"runtime:removed": [data: { projectId: string; projectName: string }];
/** Emitted when a runtime is restarted */
"project:runtime-restarted": [data: { projectId: string; projectName: string; isolationMode: import("@fusion/core").IsolationMode; reason?: string }];
}
export interface ProjectRuntimeRestartBlockedError {
kind: "active_tasks";
count: number;
}
export function isProjectRuntimeRestartBlockedError(error: unknown): error is ProjectRuntimeRestartBlockedError {
if (!error || typeof error !== "object") return false;
const candidate = error as { kind?: unknown; count?: unknown };
return candidate.kind === "active_tasks" && typeof candidate.count === "number";
}
/**
@@ -463,6 +478,43 @@ export class ProjectManager extends EventEmitter<ProjectManagerEvents> {
}
}
async restartProjectRuntime(projectId: string, options?: { reason?: string; force?: boolean }): Promise<void> {
const runtime = this.runtimes.get(projectId);
if (!runtime) {
throw new Error(`Runtime not found for project ${projectId}`);
}
const metrics = runtime.getMetrics();
if (!options?.force && metrics.inFlightTasks > 0) {
throw { kind: "active_tasks", count: metrics.inFlightTasks } satisfies ProjectRuntimeRestartBlockedError;
}
const project = await this.centralCore.getProject(projectId);
if (!project) {
throw new Error(`Project not found: ${projectId}`);
}
const workingDirectory = await this.centralCore.resolveLocalProjectWorkingDirectory(projectId);
await this.removeProject(projectId);
await this.addProject({
projectId,
workingDirectory,
isolationMode: project.isolationMode,
maxConcurrent: project.settings?.maxConcurrent ?? 2,
maxWorktrees: project.settings?.maxWorktrees ?? 4,
settings: project.settings,
});
this.emit("project:runtime-restarted", {
projectId,
projectName: project.name,
isolationMode: project.isolationMode,
reason: options?.reason,
});
}
/**
* Stop all runtimes and clean up.
*/

View File

@@ -999,64 +999,58 @@ export class Scheduler {
this.wasNodeDispatchValidationBlocked.delete(task.id);
}
// Enforce unavailable-node policy
// Enforce unavailable-node policy + owning-node handoff policy
if (effectiveNode.nodeId !== undefined && this.options.nodeHealthMonitor) {
let skipUnavailableNodePolicy = false;
if (freshTask.checkoutNodeId) {
if (freshTask.checkoutNodeId && freshTask.checkedOutBy) {
const ownerNodeHealth = this.options.nodeHealthMonitor.getNodeHealth(freshTask.checkoutNodeId);
if (ownerNodeHealth === "offline" || ownerNodeHealth === "error") {
const handoffDecision = decideOwningNodeHandoff({
task: freshTask,
ownerNodeId: freshTask.checkoutNodeId,
ownerNodeHealth,
localNodeId: settings.defaultNodeId ?? "local",
localNodeId: "local",
handoffPolicy: settings.owningNodeHandoffPolicy,
});
if (handoffDecision.action === "park") {
if (!this.wasNodeBlocked.has(task.id)) {
this.wasNodeBlocked.add(task.id);
schedulerLog.log(`Task ${task.id} dispatch blocked — ${handoffDecision.reason}`);
await this.store.logEntry(task.id, handoffDecision.reason);
const reason = `Owning-node handoff parked dispatch: ${handoffDecision.reason}`;
schedulerLog.log(`Task ${task.id} dispatch blocked — ${reason}`);
await this.store.logEntry(task.id, reason);
}
continue;
}
await this.store.logEntry(task.id, `Owning-node handoff applied: ${handoffDecision.reason}`);
if (handoffDecision.action === "reassign-local") {
effectiveNode = { nodeId: undefined, source: "local" };
await this.store.logEntry(task.id, `Owner handoff: ${handoffDecision.reason}`);
} else if (handoffDecision.action === "reassign-any") {
skipUnavailableNodePolicy = true;
await this.store.logEntry(task.id, `Owner handoff: ${handoffDecision.reason}`);
}
}
}
if (!skipUnavailableNodePolicy) {
const nodeHealth = this.options.nodeHealthMonitor.getNodeHealth(effectiveNode.nodeId);
const decision = applyUnavailableNodePolicy({
effectiveNode,
nodeHealth,
policy: settings.unavailableNodePolicy,
});
const nodeHealth = this.options.nodeHealthMonitor.getNodeHealth(effectiveNode.nodeId);
const decision = applyUnavailableNodePolicy({
effectiveNode,
nodeHealth,
policy: settings.unavailableNodePolicy,
});
if (!decision.allowed) {
if (!this.wasNodeBlocked.has(task.id)) {
this.wasNodeBlocked.add(task.id);
schedulerLog.log(`Task ${task.id} dispatch blocked — ${decision.reason}`);
await this.store.logEntry(task.id, decision.reason);
}
continue;
}
this.wasNodeBlocked.delete(task.id);
if (decision.fallbackToLocal) {
schedulerLog.log(`Task ${task.id} falling back to local — ${decision.reason}`);
if (!decision.allowed) {
if (!this.wasNodeBlocked.has(task.id)) {
this.wasNodeBlocked.add(task.id);
schedulerLog.log(`Task ${task.id} dispatch blocked — ${decision.reason}`);
await this.store.logEntry(task.id, decision.reason);
effectiveNode = { nodeId: undefined, source: "local" };
}
continue;
}
this.wasNodeBlocked.delete(task.id);
if (decision.fallbackToLocal) {
schedulerLog.log(`Task ${task.id} falling back to local — ${decision.reason}`);
await this.store.logEntry(task.id, decision.reason);
effectiveNode = { nodeId: undefined, source: "local" };
}
}