feat(FN-1079): add remote node runtime orchestration

- Add RemoteNodeClient and RemoteNodeRuntime with comprehensive lifecycle, status, and metrics test coverage
- Route projects by node assignment in ProjectManager and integrate remote runtime handling in HybridExecutor
- Introduce NodeHealthMonitor and wire diagnostic logging/export updates for remote node health tracking
- Harden remote runtime shutdown checks and type-safety paths surfaced during review feedback
This commit is contained in:
gsxdsm
2026-04-07 22:56:09 -07:00
parent 56c07aa4de
commit 55f17f3157
13 changed files with 1978 additions and 59 deletions

View File

@@ -6,64 +6,106 @@ import type { ProjectRuntimeConfig } from "../project-runtime.js";
// Mock the ProjectManager
const mockRuntimes = new Map();
const mockProjectIds: string[] = [];
const mockProjectManagerInstances: Array<{
addProject: ReturnType<typeof vi.fn>;
removeProject: ReturnType<typeof vi.fn>;
getRuntime: ReturnType<typeof vi.fn>;
listRuntimes: ReturnType<typeof vi.fn>;
getProjectIds: ReturnType<typeof vi.fn>;
getGlobalMetrics: ReturnType<typeof vi.fn>;
acquireGlobalSlot: ReturnType<typeof vi.fn>;
releaseGlobalSlot: ReturnType<typeof vi.fn>;
stopAll: ReturnType<typeof vi.fn>;
on: ReturnType<typeof vi.fn>;
}> = [];
vi.mock("../project-manager.js", () => ({
ProjectManager: vi.fn().mockImplementation(() => ({
addProject: vi.fn().mockImplementation((config: ProjectRuntimeConfig) => {
const runtime = {
start: vi.fn().mockResolvedValue(undefined),
stop: vi.fn().mockResolvedValue(undefined),
getStatus: vi.fn().mockReturnValue("active"),
getTaskStore: vi.fn(),
getScheduler: vi.fn(),
getMetrics: vi.fn().mockReturnValue({
inFlightTasks: 0,
activeAgents: 0,
lastActivityAt: new Date().toISOString(),
}),
on: vi.fn().mockReturnThis(),
};
mockRuntimes.set(config.projectId, runtime);
mockProjectIds.push(config.projectId);
return Promise.resolve(runtime);
}),
removeProject: vi.fn().mockImplementation((projectId: string) => {
mockRuntimes.delete(projectId);
const index = mockProjectIds.indexOf(projectId);
if (index > -1) mockProjectIds.splice(index, 1);
return Promise.resolve(undefined);
}),
getRuntime: vi.fn().mockImplementation((projectId: string) => {
return mockRuntimes.get(projectId);
}),
listRuntimes: vi.fn().mockImplementation(() => {
return Array.from(mockRuntimes.values());
}),
getProjectIds: vi.fn().mockImplementation(() => {
return [...mockProjectIds];
}),
getGlobalMetrics: vi.fn().mockResolvedValue({
totalInFlightTasks: 0,
totalActiveAgents: 0,
runtimeCountByStatus: {
active: 0,
paused: 0,
errored: 0,
stopped: 0,
starting: 0,
stopping: 0,
},
totalRuntimes: 0,
}),
acquireGlobalSlot: vi.fn().mockResolvedValue(true),
releaseGlobalSlot: vi.fn().mockResolvedValue(undefined),
stopAll: vi.fn().mockImplementation(() => {
mockRuntimes.clear();
mockProjectIds.length = 0;
return Promise.resolve(undefined);
}),
on: vi.fn().mockReturnThis(),
})),
ProjectManager: vi.fn().mockImplementation(() => {
const instance = {
addProject: vi.fn().mockImplementation((config: ProjectRuntimeConfig) => {
const runtime = {
start: vi.fn().mockResolvedValue(undefined),
stop: vi.fn().mockResolvedValue(undefined),
getStatus: vi.fn().mockReturnValue("active"),
getTaskStore: vi.fn(),
getScheduler: vi.fn(),
getMetrics: vi.fn().mockReturnValue({
inFlightTasks: 0,
activeAgents: 0,
lastActivityAt: new Date().toISOString(),
}),
on: vi.fn().mockReturnThis(),
};
mockRuntimes.set(config.projectId, runtime);
mockProjectIds.push(config.projectId);
return Promise.resolve(runtime);
}),
removeProject: vi.fn().mockImplementation((projectId: string) => {
mockRuntimes.delete(projectId);
const index = mockProjectIds.indexOf(projectId);
if (index > -1) mockProjectIds.splice(index, 1);
return Promise.resolve(undefined);
}),
getRuntime: vi.fn().mockImplementation((projectId: string) => {
return mockRuntimes.get(projectId);
}),
listRuntimes: vi.fn().mockImplementation(() => {
return Array.from(mockRuntimes.values());
}),
getProjectIds: vi.fn().mockImplementation(() => {
return [...mockProjectIds];
}),
getGlobalMetrics: vi.fn().mockResolvedValue({
totalInFlightTasks: 0,
totalActiveAgents: 0,
runtimeCountByStatus: {
active: 0,
paused: 0,
errored: 0,
stopped: 0,
starting: 0,
stopping: 0,
},
totalRuntimes: 0,
}),
acquireGlobalSlot: vi.fn().mockResolvedValue(true),
releaseGlobalSlot: vi.fn().mockResolvedValue(undefined),
stopAll: vi.fn().mockImplementation(() => {
mockRuntimes.clear();
mockProjectIds.length = 0;
return Promise.resolve(undefined);
}),
on: vi.fn().mockReturnThis(),
};
mockProjectManagerInstances.push(instance);
return instance;
}),
}));
const mockNodeHealthMonitorInstances: Array<{
start: ReturnType<typeof vi.fn>;
stop: ReturnType<typeof vi.fn>;
checkAllNodes: ReturnType<typeof vi.fn>;
}> = [];
vi.mock("../node-health-monitor.js", () => ({
NodeHealthMonitor: vi.fn().mockImplementation(() => {
const instance = {
start: vi.fn().mockResolvedValue(undefined),
stop: vi.fn().mockResolvedValue(undefined),
checkAllNodes: vi.fn().mockResolvedValue({
checked: 0,
online: 0,
offline: 0,
error: 0,
connecting: 0,
}),
};
mockNodeHealthMonitorInstances.push(instance);
return instance;
}),
}));
describe("HybridExecutor", () => {
@@ -112,6 +154,8 @@ describe("HybridExecutor", () => {
vi.clearAllMocks();
mockRuntimes.clear();
mockProjectIds.length = 0;
mockProjectManagerInstances.length = 0;
mockNodeHealthMonitorInstances.length = 0;
});
describe("initialization", () => {
@@ -129,6 +173,30 @@ describe("HybridExecutor", () => {
expect(mockCentralCore.listProjects).toHaveBeenCalled();
});
it("should forward remote-node assigned projects to ProjectManager for routing", async () => {
const remoteAssignedProject: RegisteredProject = {
...mockProject,
id: "proj_remote_1",
name: "Remote Assigned Project",
status: "active",
nodeId: "node_remote_1",
};
(mockCentralCore.listProjects as ReturnType<typeof vi.fn>).mockResolvedValue([
remoteAssignedProject,
]);
await executor.initialize();
const manager = mockProjectManagerInstances[0];
expect(manager?.addProject).toHaveBeenCalledWith(
expect.objectContaining({
projectId: "proj_remote_1",
isolationMode: "in-process",
workingDirectory: "/tmp/test-project",
})
);
});
it("should setup CentralCore listeners on initialize", async () => {
await executor.initialize();
expect(mockCentralCore.on).toHaveBeenCalledWith(
@@ -145,12 +213,22 @@ describe("HybridExecutor", () => {
);
});
it("should create and start node health monitor on initialize", async () => {
await executor.initialize();
expect(mockNodeHealthMonitorInstances).toHaveLength(1);
expect(mockNodeHealthMonitorInstances[0]?.start).toHaveBeenCalledTimes(1);
expect(executor.getNodeHealthMonitor()).toBe(mockNodeHealthMonitorInstances[0]);
});
it("should be idempotent (initialize multiple times)", async () => {
await executor.initialize();
const callCount = (mockCentralCore.listProjects as ReturnType<typeof vi.fn>).mock.calls.length;
await executor.initialize();
expect(mockCentralCore.listProjects).toHaveBeenCalledTimes(callCount);
expect(mockNodeHealthMonitorInstances).toHaveLength(1);
expect(mockNodeHealthMonitorInstances[0]?.start).toHaveBeenCalledTimes(1);
});
});
@@ -319,6 +397,21 @@ describe("HybridExecutor", () => {
"project:updated"
);
});
it("should stop node health monitor before stopping runtimes", async () => {
const monitor = mockNodeHealthMonitorInstances[0];
const manager = mockProjectManagerInstances[0];
await executor.shutdown();
expect(monitor?.stop).toHaveBeenCalledTimes(1);
expect(manager?.stopAll).toHaveBeenCalledTimes(1);
const monitorStopOrder = monitor?.stop.mock.invocationCallOrder[0] ?? Number.MAX_SAFE_INTEGER;
const stopAllOrder = manager?.stopAll.mock.invocationCallOrder[0] ?? Number.MIN_SAFE_INTEGER;
expect(monitorStopOrder).toBeLessThan(stopAllOrder);
expect(executor.getNodeHealthMonitor()).toBeNull();
});
});
describe("updateProject", () => {

View File

@@ -1,6 +1,7 @@
import { EventEmitter } from "node:events";
import type { Task, CentralCore, RegisteredProject } from "@fusion/core";
import { ProjectManager } from "./project-manager.js";
import { NodeHealthMonitor } from "./node-health-monitor.js";
import type {
ProjectRuntime,
ProjectRuntimeConfig,
@@ -133,6 +134,7 @@ export interface HybridExecutorOptions {
*/
export class HybridExecutor extends EventEmitter<HybridExecutorEvents> {
private projectManager: ProjectManager;
private nodeHealthMonitor: NodeHealthMonitor | null = null;
private initialized = false;
/**
@@ -197,6 +199,10 @@ export class HybridExecutor extends EventEmitter<HybridExecutorEvents> {
// Listen for CentralCore project events
this.setupCentralCoreListeners();
// Start remote node health monitoring after project runtimes are loaded.
this.nodeHealthMonitor = new NodeHealthMonitor(this.centralCore);
await this.nodeHealthMonitor.start();
this.initialized = true;
hybridExecutorLog.log("HybridExecutor initialized");
}
@@ -318,6 +324,13 @@ export class HybridExecutor extends EventEmitter<HybridExecutorEvents> {
return this.projectManager.getGlobalMetrics();
}
/**
* Get the optional node health monitor instance.
*/
getNodeHealthMonitor(): NodeHealthMonitor | null {
return this.nodeHealthMonitor;
}
/**
* Acquire a global concurrency slot.
*
@@ -355,6 +368,12 @@ export class HybridExecutor extends EventEmitter<HybridExecutorEvents> {
this.centralCore.removeAllListeners("project:unregistered");
this.centralCore.removeAllListeners("project:updated");
// Stop node health monitor before shutting down runtimes.
if (this.nodeHealthMonitor) {
await this.nodeHealthMonitor.stop();
this.nodeHealthMonitor = null;
}
// Stop all runtimes
await this.projectManager.stopAll();

View File

@@ -19,6 +19,9 @@ export { StuckTaskDetector, type StuckTaskDetectorOptions, type DisposableSessio
export { TokenCapDetector, type TokenCapCheckResult } from "./token-cap-detector.js";
export { SelfHealingManager, type SelfHealingOptions } from "./self-healing.js";
export { ProjectManager } from "./project-manager.js";
export { NodeHealthMonitor } from "./node-health-monitor.js";
export { RemoteNodeClient } from "./runtimes/remote-node-client.js";
export { RemoteNodeRuntime, type RemoteNodeRuntimeConfig } from "./runtimes/remote-node-runtime.js";
export { StepSessionExecutor } from "./step-session-executor.js";
export type { StepResult, ParallelWave, StepSessionExecutorOptions } from "./step-session-executor.js";
// Multi-project runtime types

View File

@@ -1,5 +1,14 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { createLogger, schedulerLog, executorLog, triageLog, mergerLog, worktreePoolLog, reviewerLog } from "./logger.js";
import {
createLogger,
schedulerLog,
executorLog,
triageLog,
mergerLog,
worktreePoolLog,
reviewerLog,
remoteNodeLog,
} from "./logger.js";
describe("createLogger", () => {
let logSpy: ReturnType<typeof vi.spyOn>;
@@ -71,5 +80,8 @@ describe("createLogger", () => {
reviewerLog.log("review");
expect(logSpy).toHaveBeenCalledWith("[reviewer] review");
remoteNodeLog.log("stream");
expect(logSpy).toHaveBeenCalledWith("[remote-node] stream");
});
});

View File

@@ -82,3 +82,9 @@ export const autopilotLog = createLogger("autopilot");
/** Logger for the heartbeat execution subsystem. */
export const heartbeatLog = createLogger("heartbeat");
/** Logger for remote node runtime/client subsystems. */
export const remoteNodeLog = createLogger("remote-node");
/** Logger for periodic node health monitor subsystem. */
export const nodeHealthMonitorLog = createLogger("node-health-monitor");

View File

@@ -0,0 +1,137 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { CentralCore, NodeConfig } from "@fusion/core";
import { NodeHealthMonitor } from "./node-health-monitor.js";
const NOW = "2026-04-08T00:00:00.000Z";
function createNode(overrides: Partial<NodeConfig>): NodeConfig {
return {
id: "node-id",
name: "Node",
type: "remote",
status: "online",
maxConcurrent: 4,
createdAt: NOW,
updatedAt: NOW,
...overrides,
};
}
describe("NodeHealthMonitor", () => {
let mockCentralCore: CentralCore;
let listNodesMock: ReturnType<typeof vi.fn>;
let checkNodeHealthMock: ReturnType<typeof vi.fn>;
let monitor: NodeHealthMonitor;
let warnSpy: ReturnType<typeof vi.spyOn>;
let logSpy: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
vi.useFakeTimers();
warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
listNodesMock = vi.fn().mockResolvedValue([
createNode({ id: "node-local", name: "Local Node", type: "local", status: "online" }),
createNode({ id: "node-remote", name: "Remote Node", type: "remote", status: "online" }),
]);
checkNodeHealthMock = vi.fn().mockResolvedValue("online");
mockCentralCore = {
listNodes: listNodesMock,
checkNodeHealth: checkNodeHealthMock,
} as unknown as CentralCore;
monitor = new NodeHealthMonitor(mockCentralCore, { checkIntervalMs: 1_000 });
});
afterEach(async () => {
await monitor.stop();
vi.useRealTimers();
warnSpy.mockRestore();
logSpy.mockRestore();
vi.clearAllMocks();
});
it("start() sets interval and checks only remote nodes", async () => {
await monitor.start();
await vi.advanceTimersByTimeAsync(1_000);
expect(listNodesMock).toHaveBeenCalled();
expect(checkNodeHealthMock).toHaveBeenCalledTimes(1);
expect(checkNodeHealthMock).toHaveBeenCalledWith("node-remote");
});
it("stop() clears interval", async () => {
await monitor.start();
await vi.advanceTimersByTimeAsync(1_000);
expect(checkNodeHealthMock).toHaveBeenCalledTimes(1);
await monitor.stop();
checkNodeHealthMock.mockClear();
await vi.advanceTimersByTimeAsync(2_000);
expect(checkNodeHealthMock).not.toHaveBeenCalled();
});
it("checkAllNodes() checks each remote node and skips local nodes", async () => {
await monitor.start();
const summary = await monitor.checkAllNodes();
expect(checkNodeHealthMock).toHaveBeenCalledTimes(1);
expect(checkNodeHealthMock).toHaveBeenCalledWith("node-remote");
expect(summary).toEqual({
checked: 1,
online: 1,
offline: 0,
error: 0,
connecting: 0,
});
});
it("logs warning when node transitions from online to offline", async () => {
checkNodeHealthMock.mockResolvedValue("offline");
await monitor.start();
await monitor.checkAllNodes();
expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining("[node-health-monitor] Remote node Remote Node (node-remote) degraded")
);
expect(monitor.getNodeHealth("node-remote")).toBe("offline");
});
it("logs recovery when node transitions back to online", async () => {
checkNodeHealthMock.mockResolvedValueOnce("offline").mockResolvedValueOnce("online");
await monitor.start();
await monitor.checkAllNodes();
await monitor.checkAllNodes();
expect(logSpy).toHaveBeenCalledWith(
expect.stringContaining("[node-health-monitor] Remote node Remote Node (node-remote) recovered")
);
expect(monitor.getNodeHealth("node-remote")).toBe("online");
});
it("is a no-op when no remote nodes are registered", async () => {
listNodesMock.mockResolvedValue([
createNode({ id: "node-local-only", name: "Local Only", type: "local", status: "online" }),
]);
await monitor.start();
const summary = await monitor.checkAllNodes();
expect(checkNodeHealthMock).not.toHaveBeenCalled();
expect(summary).toEqual({
checked: 0,
online: 0,
offline: 0,
error: 0,
connecting: 0,
});
});
});

View File

@@ -0,0 +1,153 @@
import type { CentralCore, NodeStatus } from "@fusion/core";
import { nodeHealthMonitorLog } from "./logger.js";
export interface NodeHealthMonitorOptions {
checkIntervalMs?: number;
}
export interface NodeHealthCheckSummary {
checked: number;
online: number;
offline: number;
error: number;
connecting: number;
}
export class NodeHealthMonitor {
private readonly checkIntervalMs: number;
private interval: ReturnType<typeof setInterval> | null = null;
private running = false;
private lastKnownStatus = new Map<string, NodeStatus>();
private activeCheck: Promise<NodeHealthCheckSummary> | null = null;
constructor(
private readonly centralCore: CentralCore,
options: NodeHealthMonitorOptions = {}
) {
this.checkIntervalMs = options.checkIntervalMs ?? 60_000;
}
async start(): Promise<void> {
if (this.running) {
return;
}
this.running = true;
const nodes = await this.centralCore.listNodes();
for (const node of nodes) {
if (node.type === "remote") {
this.lastKnownStatus.set(node.id, node.status);
}
}
this.interval = setInterval(() => {
void this.checkAllNodes();
}, this.checkIntervalMs);
nodeHealthMonitorLog.log(
`NodeHealthMonitor started (${this.lastKnownStatus.size} remote nodes, interval=${this.checkIntervalMs}ms)`
);
}
async stop(): Promise<void> {
this.running = false;
if (this.interval) {
clearInterval(this.interval);
this.interval = null;
}
if (this.activeCheck) {
await this.activeCheck.catch(() => {
// Best-effort: pending check errors are already logged.
});
}
nodeHealthMonitorLog.log("NodeHealthMonitor stopped");
}
async checkAllNodes(): Promise<NodeHealthCheckSummary> {
if (!this.running) {
return {
checked: 0,
online: 0,
offline: 0,
error: 0,
connecting: 0,
};
}
if (this.activeCheck) {
return this.activeCheck;
}
this.activeCheck = this.runCheckAllNodes();
try {
return await this.activeCheck;
} finally {
this.activeCheck = null;
}
}
getNodeHealth(nodeId: string): NodeStatus | undefined {
return this.lastKnownStatus.get(nodeId);
}
private async runCheckAllNodes(): Promise<NodeHealthCheckSummary> {
const nodes = await this.centralCore.listNodes();
const remoteNodes = nodes.filter((node) => node.type === "remote");
if (remoteNodes.length === 0) {
return {
checked: 0,
online: 0,
offline: 0,
error: 0,
connecting: 0,
};
}
const summary: NodeHealthCheckSummary = {
checked: 0,
online: 0,
offline: 0,
error: 0,
connecting: 0,
};
for (const node of remoteNodes) {
try {
const previousStatus = this.lastKnownStatus.get(node.id) ?? node.status;
const nextStatus = await this.centralCore.checkNodeHealth(node.id);
this.lastKnownStatus.set(node.id, nextStatus);
summary.checked += 1;
summary[nextStatus] += 1;
if (previousStatus !== nextStatus) {
if (previousStatus === "online" && (nextStatus === "offline" || nextStatus === "error")) {
nodeHealthMonitorLog.warn(
`Remote node ${node.name} (${node.id}) degraded: ${previousStatus}${nextStatus}`
);
} else if (previousStatus !== "online" && nextStatus === "online") {
nodeHealthMonitorLog.log(
`Remote node ${node.name} (${node.id}) recovered: ${previousStatus} → online`
);
}
}
} catch (error) {
this.lastKnownStatus.set(node.id, "error");
summary.checked += 1;
summary.error += 1;
nodeHealthMonitorLog.warn(
`Failed to check node ${node.name} (${node.id}) health: ${
error instanceof Error ? error.message : String(error)
}`
);
}
}
return summary;
}
}

View File

@@ -1,5 +1,8 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import type { CentralCore, RegisteredProject, Task } from "@fusion/core";
import { InProcessRuntime } from "./runtimes/in-process-runtime.js";
import { ChildProcessRuntime } from "./runtimes/child-process-runtime.js";
import { RemoteNodeRuntime } from "./runtimes/remote-node-runtime.js";
import { ProjectManager } from "./project-manager.js";
import type { ProjectRuntimeConfig } from "./project-runtime.js";
@@ -40,6 +43,26 @@ vi.mock("./runtimes/child-process-runtime.js", () => ({
})),
}));
vi.mock("./runtimes/remote-node-runtime.js", () => ({
RemoteNodeRuntime: vi.fn().mockImplementation(() => ({
start: vi.fn().mockResolvedValue(undefined),
stop: vi.fn().mockResolvedValue(undefined),
getStatus: vi.fn().mockReturnValue("active"),
getTaskStore: vi.fn().mockImplementation(() => {
throw new Error("TaskStore not accessible for remote node runtime");
}),
getScheduler: vi.fn().mockImplementation(() => {
throw new Error("Scheduler not accessible for remote node runtime");
}),
getMetrics: vi.fn().mockReturnValue({
inFlightTasks: 0,
activeAgents: 0,
lastActivityAt: new Date().toISOString(),
}),
on: vi.fn().mockReturnThis(),
})),
}));
describe("ProjectManager", () => {
let manager: ProjectManager;
let mockCentralCore: CentralCore;
@@ -56,6 +79,7 @@ describe("ProjectManager", () => {
beforeEach(() => {
mockCentralCore = {
getProject: vi.fn().mockResolvedValue(mockProject),
getNode: vi.fn().mockResolvedValue(undefined),
getGlobalConcurrencyState: vi.fn().mockResolvedValue({
globalMaxConcurrent: 4,
currentlyActive: 0,
@@ -143,6 +167,84 @@ describe("ProjectManager", () => {
{ status: "active" }
);
});
it("routes to RemoteNodeRuntime when assigned node is remote", async () => {
(mockCentralCore.getProject as ReturnType<typeof vi.fn>).mockResolvedValue({
...mockProject,
nodeId: "node_remote_1",
});
(mockCentralCore.getNode as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "node_remote_1",
name: "Remote 1",
type: "remote",
url: "https://remote.example.com",
apiKey: "remote-token",
status: "online",
maxConcurrent: 4,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
});
await manager.addProject(testConfig);
expect(RemoteNodeRuntime).toHaveBeenCalledWith({
nodeConfig: expect.objectContaining({ id: "node_remote_1", type: "remote" }),
projectId: "proj_test123",
projectName: "Test Project",
});
});
it("routes to InProcessRuntime when assigned node is local", async () => {
(mockCentralCore.getProject as ReturnType<typeof vi.fn>).mockResolvedValue({
...mockProject,
nodeId: "node_local_1",
});
(mockCentralCore.getNode as ReturnType<typeof vi.fn>).mockResolvedValue({
id: "node_local_1",
name: "Local 1",
type: "local",
status: "online",
maxConcurrent: 4,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
});
await manager.addProject(testConfig);
expect(InProcessRuntime).toHaveBeenCalled();
expect(RemoteNodeRuntime).not.toHaveBeenCalled();
});
it("routes to InProcessRuntime when no node assignment exists", async () => {
(mockCentralCore.getProject as ReturnType<typeof vi.fn>).mockResolvedValue({
...mockProject,
nodeId: undefined,
});
await manager.addProject(testConfig);
expect(InProcessRuntime).toHaveBeenCalled();
expect(mockCentralCore.getNode).not.toHaveBeenCalled();
});
it("falls back to InProcessRuntime and logs warning when assigned node is missing", async () => {
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
(mockCentralCore.getProject as ReturnType<typeof vi.fn>).mockResolvedValue({
...mockProject,
nodeId: "node_missing",
});
(mockCentralCore.getNode as ReturnType<typeof vi.fn>).mockResolvedValue(undefined);
await manager.addProject(testConfig);
expect(InProcessRuntime).toHaveBeenCalled();
expect(RemoteNodeRuntime).not.toHaveBeenCalled();
expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining("[project-manager] Assigned node node_missing not found")
);
warnSpy.mockRestore();
});
});
describe("removeProject", () => {

View File

@@ -2,6 +2,7 @@ import { EventEmitter } from "node:events";
import type { Task, CentralCore, RegisteredProject } from "@fusion/core";
import { InProcessRuntime } from "./runtimes/in-process-runtime.js";
import { ChildProcessRuntime } from "./runtimes/child-process-runtime.js";
import { RemoteNodeRuntime } from "./runtimes/remote-node-runtime.js";
import { AgentSemaphore } from "./concurrency.js";
import type {
ProjectRuntime,
@@ -163,8 +164,28 @@ export class ProjectManager extends EventEmitter<ProjectManagerEvents> {
if (config.isolationMode === "child-process") {
runtime = new ChildProcessRuntime(config, this.centralCore);
} else {
// Default to in-process
runtime = new InProcessRuntime(config, this.centralCore);
let assignedNode = undefined;
if (project.nodeId) {
assignedNode = await this.centralCore.getNode(project.nodeId);
if (!assignedNode) {
projectManagerLog.warn(
`Assigned node ${project.nodeId} not found for project ${project.id}; falling back to InProcessRuntime`
);
}
}
if (assignedNode?.type === "remote") {
runtime = new RemoteNodeRuntime({
nodeConfig: assignedNode,
projectId: config.projectId,
projectName: project.name,
});
} else {
// Default to local in-process runtime (includes unassigned + local-node assigned)
runtime = new InProcessRuntime(config, this.centralCore);
}
}
// Set up event forwarding with project attribution

View File

@@ -0,0 +1,323 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { RuntimeMetrics } from "../project-runtime.js";
import { RemoteNodeClient } from "./remote-node-client.js";
const BASE_URL = "https://node.example.com";
const API_KEY = "secret-token";
describe("RemoteNodeClient", () => {
const originalFetch = globalThis.fetch;
beforeEach(() => {
vi.useRealTimers();
});
afterEach(() => {
globalThis.fetch = originalFetch;
vi.restoreAllMocks();
vi.useRealTimers();
});
it("health() parses successful response and sends auth header", async () => {
const fetchMock = vi.fn().mockResolvedValue(
new Response(JSON.stringify({ status: "ok", version: "1.0.0", uptime: 123 }), {
status: 200,
headers: { "content-type": "application/json" },
})
);
globalThis.fetch = fetchMock as unknown as typeof fetch;
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
const health = await client.health();
expect(health).toEqual({ status: "ok", version: "1.0.0", uptime: 123 });
expect(fetchMock).toHaveBeenCalledWith(`${BASE_URL}/api/health`, expect.objectContaining({
method: "GET",
headers: expect.objectContaining({
Authorization: `Bearer ${API_KEY}`,
}),
}));
});
it("getMetrics() parses runtime metrics", async () => {
const metrics: RuntimeMetrics = {
inFlightTasks: 4,
activeAgents: 2,
lastActivityAt: "2026-04-08T00:00:00.000Z",
};
globalThis.fetch = vi.fn().mockResolvedValue(
new Response(JSON.stringify(metrics), {
status: 200,
headers: { "content-type": "application/json" },
})
) as unknown as typeof fetch;
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
await expect(client.getMetrics()).resolves.toEqual(metrics);
});
it("createTask() sends POST with JSON body", async () => {
const createdTask = {
id: "KB-001",
description: "Create me",
column: "todo",
dependencies: [],
steps: [],
currentStep: 0,
status: "pending",
log: [],
attachments: [],
createdAt: "2026-04-08T00:00:00.000Z",
updatedAt: "2026-04-08T00:00:00.000Z",
size: "M",
reviewLevel: 1,
};
const fetchMock = vi.fn().mockResolvedValue(
new Response(JSON.stringify(createdTask), {
status: 200,
headers: { "content-type": "application/json" },
})
);
globalThis.fetch = fetchMock as unknown as typeof fetch;
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
await client.createTask({ description: "Create me" });
const options = fetchMock.mock.calls[0]?.[1] as RequestInit;
expect(fetchMock).toHaveBeenCalledWith(`${BASE_URL}/api/tasks`, expect.any(Object));
expect(options.method).toBe("POST");
expect(options.headers).toEqual(expect.objectContaining({
"Content-Type": "application/json",
Authorization: `Bearer ${API_KEY}`,
}));
expect(options.body).toBe(JSON.stringify({ description: "Create me" }));
});
it("listTasks() sends optional query params", async () => {
const fetchMock = vi.fn().mockResolvedValue(
new Response(JSON.stringify([]), {
status: 200,
headers: { "content-type": "application/json" },
})
);
globalThis.fetch = fetchMock as unknown as typeof fetch;
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
await client.listTasks({ column: "in-progress", limit: 10 });
expect(fetchMock).toHaveBeenCalledWith(
`${BASE_URL}/api/tasks?column=in-progress&limit=10`,
expect.objectContaining({ method: "GET" })
);
});
it("executeTask() posts to execute endpoint", async () => {
const fetchMock = vi.fn().mockResolvedValue(
new Response(JSON.stringify({ acknowledged: true }), {
status: 200,
headers: { "content-type": "application/json" },
})
);
globalThis.fetch = fetchMock as unknown as typeof fetch;
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
const result = await client.executeTask("KB-123");
expect(result).toEqual({ acknowledged: true });
expect(fetchMock).toHaveBeenCalledWith(
`${BASE_URL}/api/tasks/KB-123/execute`,
expect.objectContaining({ method: "POST" })
);
});
it("streamEvents() yields parsed events from SSE stream", async () => {
const sseBody = [
"event: task:created",
'data: {"type":"task:created","payload":{"id":"KB-1"},"timestamp":"2026-04-08T00:00:00.000Z"}',
"",
"event: task:updated",
'data: {"type":"task:updated","payload":{"id":"KB-1","column":"in-progress"},"timestamp":"2026-04-08T00:01:00.000Z"}',
"",
].join("\n");
globalThis.fetch = vi.fn().mockResolvedValue(
new Response(sseBody, {
status: 200,
headers: { "content-type": "text/event-stream" },
})
) as unknown as typeof fetch;
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
const events: unknown[] = [];
for await (const event of client.streamEvents()) {
events.push(event);
}
expect(events).toEqual([
{
type: "task:created",
payload: { id: "KB-1" },
timestamp: "2026-04-08T00:00:00.000Z",
},
{
type: "task:updated",
payload: { id: "KB-1", column: "in-progress" },
timestamp: "2026-04-08T00:01:00.000Z",
},
]);
});
it("retries on network errors", async () => {
const fetchMock = vi
.fn()
.mockRejectedValueOnce(new TypeError("network down"))
.mockResolvedValueOnce(
new Response(JSON.stringify({ status: "ok", version: "1.0.0", uptime: 123 }), {
status: 200,
headers: { "content-type": "application/json" },
})
);
globalThis.fetch = fetchMock as unknown as typeof fetch;
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
await expect(client.health()).resolves.toEqual({
status: "ok",
version: "1.0.0",
uptime: 123,
});
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it("does not retry on 4xx responses", async () => {
const fetchMock = vi.fn().mockResolvedValue(
new Response(JSON.stringify({ error: "unauthorized" }), {
status: 401,
statusText: "Unauthorized",
headers: { "content-type": "application/json" },
})
);
globalThis.fetch = fetchMock as unknown as typeof fetch;
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
await expect(client.health()).rejects.toThrow("401 Unauthorized");
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it("retries on 5xx responses", async () => {
const fetchMock = vi
.fn()
.mockResolvedValueOnce(
new Response("server error", { status: 500, statusText: "Internal Server Error" })
)
.mockResolvedValueOnce(
new Response("server error", { status: 502, statusText: "Bad Gateway" })
)
.mockResolvedValueOnce(
new Response(JSON.stringify({ status: "ok", version: "1.0.0", uptime: 999 }), {
status: 200,
headers: { "content-type": "application/json" },
})
);
globalThis.fetch = fetchMock as unknown as typeof fetch;
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
await expect(client.health()).resolves.toEqual({
status: "ok",
version: "1.0.0",
uptime: 999,
});
expect(fetchMock).toHaveBeenCalledTimes(3);
});
it("aborts requests after timeoutMs", async () => {
vi.useFakeTimers();
const fetchMock = vi.fn().mockImplementation((_: unknown, init?: RequestInit) => {
return new Promise((_resolve, reject) => {
const signal = init?.signal;
signal?.addEventListener("abort", () => {
const abortError = new Error("aborted");
abortError.name = "AbortError";
reject(abortError);
});
});
});
globalThis.fetch = fetchMock as unknown as typeof fetch;
const client = new RemoteNodeClient({
baseUrl: BASE_URL,
apiKey: API_KEY,
timeoutMs: 5,
});
const request = client.health();
const expectation = expect(request).rejects.toThrow("timed out");
await vi.runAllTimersAsync();
await expectation;
expect(fetchMock).toHaveBeenCalledTimes(4); // initial + 3 retries
});
it("sends auth header on all request methods", async () => {
const fetchMock = vi
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ status: "ok", version: "1.0.0", uptime: 1 }), {
status: 200,
headers: { "content-type": "application/json" },
})
)
.mockResolvedValueOnce(
new Response(JSON.stringify({ inFlightTasks: 0, activeAgents: 0, lastActivityAt: "now" }), {
status: 200,
headers: { "content-type": "application/json" },
})
)
.mockResolvedValueOnce(
new Response(JSON.stringify([]), {
status: 200,
headers: { "content-type": "application/json" },
})
)
.mockResolvedValueOnce(
new Response(JSON.stringify({ acknowledged: true }), {
status: 200,
headers: { "content-type": "application/json" },
})
)
.mockResolvedValueOnce(
new Response("event: ping\ndata: {}\n\n", {
status: 200,
headers: { "content-type": "text/event-stream" },
})
);
globalThis.fetch = fetchMock as unknown as typeof fetch;
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
await client.health();
await client.getMetrics();
await client.listTasks();
await client.executeTask("KB-777");
for await (const _event of client.streamEvents()) {
// Drain one-response event stream
}
for (const call of fetchMock.mock.calls) {
const options = call[1] as RequestInit;
expect(options.headers).toEqual(
expect.objectContaining({
Authorization: `Bearer ${API_KEY}`,
})
);
}
});
});

View File

@@ -0,0 +1,441 @@
import type { Task, TaskCreateInput } from "@fusion/core";
import type { RuntimeMetrics } from "../project-runtime.js";
import { remoteNodeLog } from "../logger.js";
export interface RemoteNodeEvent {
type: string;
payload: unknown;
timestamp: string;
}
export interface RemoteNodeClientOptions {
baseUrl: string;
apiKey: string;
timeoutMs?: number;
}
export type RemoteTaskListFilter = Record<string, string | number | boolean | undefined | null>;
class RemoteNodeRequestError extends Error {
constructor(
message: string,
readonly retryable: boolean,
readonly status?: number
) {
super(message);
this.name = "RemoteNodeRequestError";
}
}
const RETRY_BASE_DELAY_MS = 1000;
const DEFAULT_MAX_RETRIES = 3;
export class RemoteNodeClient {
private readonly baseUrl: string;
private readonly apiKey: string;
private readonly timeoutMs: number;
constructor(options: RemoteNodeClientOptions) {
this.baseUrl = options.baseUrl.replace(/\/+$/, "");
this.apiKey = options.apiKey;
this.timeoutMs = options.timeoutMs ?? 30_000;
}
async health(): Promise<{ status: string; version: string; uptime: number }> {
return this.withRetry(() =>
this.requestJson<{ status: string; version: string; uptime: number }>("/api/health", {
method: "GET",
})
);
}
async getMetrics(): Promise<RuntimeMetrics> {
return this.withRetry(() =>
this.requestJson<RuntimeMetrics>("/api/metrics", {
method: "GET",
})
);
}
async createTask(input: TaskCreateInput): Promise<Task> {
return this.withRetry(() =>
this.requestJson<Task>("/api/tasks", {
method: "POST",
body: JSON.stringify(input),
})
);
}
async listTasks(filter?: RemoteTaskListFilter): Promise<Task[]> {
const query = new URLSearchParams();
if (filter) {
for (const [key, value] of Object.entries(filter)) {
if (value !== undefined && value !== null) {
query.set(key, String(value));
}
}
}
const path = query.toString().length > 0 ? `/api/tasks?${query.toString()}` : "/api/tasks";
return this.withRetry(() =>
this.requestJson<Task[]>(path, {
method: "GET",
})
);
}
async executeTask(taskId: string): Promise<{ acknowledged: boolean; [key: string]: unknown }> {
return this.withRetry(() =>
this.requestJson<{ acknowledged: boolean; [key: string]: unknown }>(
`/api/tasks/${encodeURIComponent(taskId)}/execute`,
{
method: "POST",
}
)
);
}
async *streamEvents(options?: { signal?: AbortSignal }): AsyncIterable<RemoteNodeEvent> {
const response = await this.withRetry(
() => this.openStream("/api/events/stream", options?.signal),
DEFAULT_MAX_RETRIES
);
const contentType = response.headers.get("content-type") ?? "";
if (!response.body) {
throw new Error("Remote node event stream opened without a body");
}
if (contentType.includes("text/event-stream")) {
yield* this.parseSseStream(response.body, options?.signal);
return;
}
// Fallback for long-polling endpoints that return JSON payloads.
if (contentType.includes("application/json")) {
const payload = (await response.json()) as unknown;
if (Array.isArray(payload)) {
for (const rawEvent of payload) {
yield this.normalizeEvent(rawEvent, "message");
}
} else {
yield this.normalizeEvent(payload, "message");
}
return;
}
// Generic fallback: treat each line as one JSON event.
yield* this.parseJsonLines(response.body, options?.signal);
}
private async requestJson<T>(path: string, init: RequestInit): Promise<T> {
const response = await this.fetchWithTimeout(path, {
...init,
headers: {
...this.getAuthHeaders(),
Accept: "application/json",
...(init.body ? { "Content-Type": "application/json" } : {}),
...(init.headers ?? {}),
},
});
if (!response.ok) {
await this.throwHttpError(path, response);
}
try {
return (await response.json()) as T;
} catch (error) {
throw new RemoteNodeRequestError(
`Failed to parse JSON response for ${path}: ${error instanceof Error ? error.message : String(error)}`,
false
);
}
}
private async openStream(path: string, signal?: AbortSignal): Promise<Response> {
const response = await this.fetchWithTimeout(
path,
{
method: "GET",
headers: {
...this.getAuthHeaders(),
Accept: "text/event-stream, application/json",
},
},
signal
);
if (!response.ok) {
await this.throwHttpError(path, response);
}
return response;
}
private async throwHttpError(path: string, response: Response): Promise<never> {
const responseBody = (await response.text()).trim();
const snippet = responseBody.length > 0 ? `${responseBody.slice(0, 300)}` : "";
const retryable = response.status >= 500;
throw new RemoteNodeRequestError(
`Remote node request failed (${response.status} ${response.statusText}) for ${path}${snippet}`,
retryable,
response.status
);
}
private getAuthHeaders(): Record<string, string> {
return {
Authorization: `Bearer ${this.apiKey}`,
};
}
private async fetchWithTimeout(
path: string,
init: RequestInit,
externalSignal?: AbortSignal
): Promise<Response> {
const controller = new AbortController();
let timedOut = false;
const timeout = setTimeout(() => {
timedOut = true;
controller.abort();
}, this.timeoutMs);
const onAbort = () => controller.abort(externalSignal?.reason);
if (externalSignal) {
if (externalSignal.aborted) {
clearTimeout(timeout);
throw new RemoteNodeRequestError("Request aborted", false);
}
externalSignal.addEventListener("abort", onAbort, { once: true });
}
try {
return await fetch(`${this.baseUrl}${path}`, {
...init,
signal: controller.signal,
});
} catch (error) {
if (error instanceof RemoteNodeRequestError) {
throw error;
}
if (timedOut) {
throw new RemoteNodeRequestError(
`Remote node request timed out after ${this.timeoutMs}ms (${path})`,
true
);
}
if (error instanceof Error && error.name === "AbortError") {
throw new RemoteNodeRequestError(`Remote node request aborted (${path})`, false);
}
throw new RemoteNodeRequestError(
`Remote node network error (${path}): ${error instanceof Error ? error.message : String(error)}`,
true
);
} finally {
clearTimeout(timeout);
if (externalSignal) {
externalSignal.removeEventListener("abort", onAbort);
}
}
}
private async *parseSseStream(
stream: ReadableStream<Uint8Array>,
signal?: AbortSignal
): AsyncIterable<RemoteNodeEvent> {
const reader = stream.getReader();
const decoder = new TextDecoder();
let buffer = "";
let eventType = "message";
let dataLines: string[] = [];
const flushEvent = (): RemoteNodeEvent | null => {
if (dataLines.length === 0) {
eventType = "message";
return null;
}
const data = dataLines.join("\n");
dataLines = [];
const normalized = this.normalizeEvent(data, eventType);
eventType = "message";
return normalized;
};
try {
while (true) {
if (signal?.aborted) {
break;
}
const { done, value } = await reader.read();
if (done) break;
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split(/\r?\n/);
buffer = lines.pop() ?? "";
for (const line of lines) {
if (line.length === 0) {
const event = flushEvent();
if (event) {
yield event;
}
continue;
}
if (line.startsWith(":")) {
continue;
}
const separator = line.indexOf(":");
const field = separator === -1 ? line : line.slice(0, separator);
const valuePart = separator === -1 ? "" : line.slice(separator + 1).trimStart();
if (field === "event") {
eventType = valuePart || "message";
} else if (field === "data") {
dataLines.push(valuePart);
}
}
}
if (buffer.trim().length > 0) {
dataLines.push(buffer.trim());
}
const trailingEvent = flushEvent();
if (trailingEvent) {
yield trailingEvent;
}
} finally {
reader.releaseLock();
}
}
private async *parseJsonLines(
stream: ReadableStream<Uint8Array>,
signal?: AbortSignal
): AsyncIterable<RemoteNodeEvent> {
const reader = stream.getReader();
const decoder = new TextDecoder();
let buffer = "";
try {
while (true) {
if (signal?.aborted) {
break;
}
const { done, value } = await reader.read();
if (done) break;
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split(/\r?\n/);
buffer = lines.pop() ?? "";
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) continue;
yield this.normalizeEvent(trimmed, "message");
}
}
const trailing = buffer.trim();
if (trailing.length > 0) {
yield this.normalizeEvent(trailing, "message");
}
} finally {
reader.releaseLock();
}
}
private normalizeEvent(raw: unknown, fallbackType: string): RemoteNodeEvent {
const parsed = this.tryParseJson(raw);
if (
parsed &&
typeof parsed === "object" &&
"type" in parsed &&
"timestamp" in parsed
) {
return {
type: String((parsed as { type: unknown }).type),
payload: (parsed as { payload?: unknown }).payload,
timestamp: String((parsed as { timestamp: unknown }).timestamp),
};
}
return {
type: fallbackType,
payload: parsed,
timestamp: new Date().toISOString(),
};
}
private tryParseJson(value: unknown): unknown {
if (typeof value !== "string") {
return value;
}
try {
return JSON.parse(value);
} catch {
return value;
}
}
private async withRetry<T>(fn: () => Promise<T>, maxRetries = DEFAULT_MAX_RETRIES): Promise<T> {
let attempt = 0;
while (true) {
try {
return await fn();
} catch (error) {
const isRetryable =
error instanceof RemoteNodeRequestError
? error.retryable
: this.isLikelyNetworkError(error);
if (!isRetryable || attempt >= maxRetries) {
throw error;
}
const delayMs = RETRY_BASE_DELAY_MS * 2 ** attempt;
attempt += 1;
remoteNodeLog.warn(
`Request failed, retrying in ${delayMs}ms (attempt ${attempt}/${maxRetries})`,
error
);
await this.sleep(delayMs);
}
}
}
private isLikelyNetworkError(error: unknown): boolean {
if (!(error instanceof Error)) {
return false;
}
if (error.name === "AbortError") {
return true;
}
return error instanceof TypeError;
}
private async sleep(ms: number): Promise<void> {
await new Promise<void>((resolve) => {
setTimeout(resolve, ms);
});
}
}

View File

@@ -0,0 +1,266 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { NodeConfig } from "@fusion/core";
import type { RuntimeMetrics } from "../project-runtime.js";
import { RemoteNodeRuntime } from "./remote-node-runtime.js";
const mockClientConstructor = vi.hoisted(() => vi.fn());
const mockHealth = vi.hoisted(() => vi.fn());
const mockGetMetrics = vi.hoisted(() => vi.fn());
const mockStreamEvents = vi.hoisted(() => vi.fn());
vi.mock("./remote-node-client.js", () => ({
RemoteNodeClient: vi.fn().mockImplementation((options: unknown) => {
mockClientConstructor(options);
return {
health: mockHealth,
getMetrics: mockGetMetrics,
streamEvents: mockStreamEvents,
};
}),
}));
const NOW = "2026-04-08T00:00:00.000Z";
function createNode(overrides?: Partial<NodeConfig>): NodeConfig {
return {
id: "node_remote_1",
name: "Remote Node",
type: "remote",
url: "https://remote.example.com",
apiKey: "token-123",
status: "online",
maxConcurrent: 4,
createdAt: NOW,
updatedAt: NOW,
...overrides,
};
}
async function* idleStream(signal?: AbortSignal): AsyncIterable<unknown> {
while (!signal?.aborted) {
await new Promise((resolve) => setTimeout(resolve, 5));
}
}
async function* eventStream(events: unknown[], signal?: AbortSignal): AsyncIterable<unknown> {
for (const event of events) {
yield event;
}
while (!signal?.aborted) {
await new Promise((resolve) => setTimeout(resolve, 5));
}
}
describe("RemoteNodeRuntime", () => {
beforeEach(() => {
mockClientConstructor.mockReset();
mockHealth.mockReset();
mockGetMetrics.mockReset();
mockStreamEvents.mockReset();
mockHealth.mockResolvedValue({ status: "ok", version: "1.0.0", uptime: 100 });
mockGetMetrics.mockResolvedValue({
inFlightTasks: 1,
activeAgents: 2,
lastActivityAt: NOW,
} satisfies RuntimeMetrics);
mockStreamEvents.mockImplementation(({ signal }: { signal?: AbortSignal } = {}) =>
idleStream(signal)
);
});
afterEach(async () => {
vi.clearAllMocks();
vi.useRealTimers();
});
it("start() transitions stopped -> starting -> active and starts stream", async () => {
const runtime = new RemoteNodeRuntime({
nodeConfig: createNode(),
projectId: "proj_1",
projectName: "Project 1",
});
const healthEvents: string[] = [];
runtime.on("health-changed", ({ status }) => {
healthEvents.push(status);
});
await runtime.start();
expect(runtime.getStatus()).toBe("active");
expect(healthEvents).toEqual(["starting", "active"]);
expect(mockHealth).toHaveBeenCalled();
expect(mockStreamEvents).toHaveBeenCalled();
expect(mockClientConstructor).toHaveBeenCalledWith({
baseUrl: "https://remote.example.com",
apiKey: "token-123",
});
await runtime.stop();
});
it("stop() transitions to stopped and is idempotent", async () => {
const runtime = new RemoteNodeRuntime({
nodeConfig: createNode(),
projectId: "proj_2",
projectName: "Project 2",
});
await runtime.start();
await runtime.stop();
expect(runtime.getStatus()).toBe("stopped");
await expect(runtime.stop()).resolves.toBeUndefined();
});
it("getTaskStore() throws descriptive error", () => {
const runtime = new RemoteNodeRuntime({
nodeConfig: createNode(),
projectId: "proj_3",
projectName: "Project 3",
});
expect(() => runtime.getTaskStore()).toThrow(
"TaskStore not accessible for remote node runtime"
);
});
it("getScheduler() throws descriptive error", () => {
const runtime = new RemoteNodeRuntime({
nodeConfig: createNode(),
projectId: "proj_4",
projectName: "Project 4",
});
expect(() => runtime.getScheduler()).toThrow("Scheduler not accessible for remote node runtime");
});
it("getMetrics() returns fetched metrics on success and fallback on failure", async () => {
const runtime = new RemoteNodeRuntime({
nodeConfig: createNode(),
projectId: "proj_5",
projectName: "Project 5",
});
await runtime.start();
expect(runtime.getMetrics()).toEqual({
inFlightTasks: 1,
activeAgents: 2,
lastActivityAt: NOW,
});
mockGetMetrics.mockRejectedValueOnce(new Error("metrics unavailable"));
runtime.getMetrics();
await Promise.resolve();
expect(runtime.getMetrics()).toEqual({
inFlightTasks: 0,
activeAgents: 0,
lastActivityAt: NOW,
});
await runtime.stop();
});
it("forwards remote task and error events", async () => {
const createdHandler = vi.fn();
const movedHandler = vi.fn();
const updatedHandler = vi.fn();
const errorHandler = vi.fn();
mockStreamEvents.mockImplementation(({ signal }: { signal?: AbortSignal } = {}) =>
eventStream(
[
{
type: "task:created",
payload: { id: "KB-1" },
timestamp: NOW,
},
{
type: "task:moved",
payload: { task: { id: "KB-1" }, from: "todo", to: "in-progress" },
timestamp: NOW,
},
{
type: "task:updated",
payload: { id: "KB-1", column: "done" },
timestamp: NOW,
},
{
type: "error",
payload: { message: "boom" },
timestamp: NOW,
},
],
signal
)
);
const runtime = new RemoteNodeRuntime({
nodeConfig: createNode(),
projectId: "proj_6",
projectName: "Project 6",
});
runtime.on("task:created", createdHandler);
runtime.on("task:moved", movedHandler);
runtime.on("task:updated", updatedHandler);
runtime.on("error", errorHandler);
await runtime.start();
await vi.waitFor(() => {
expect(createdHandler).toHaveBeenCalledWith({ id: "KB-1" });
expect(movedHandler).toHaveBeenCalledWith({
task: { id: "KB-1" },
from: "todo",
to: "in-progress",
});
expect(updatedHandler).toHaveBeenCalledWith({ id: "KB-1", column: "done" });
expect(errorHandler).toHaveBeenCalledWith(expect.any(Error));
});
await runtime.stop();
});
it("reconnects when stream ends unexpectedly and transitions to errored after max attempts", async () => {
mockStreamEvents.mockImplementation(async function* () {
// Immediate end to force reconnect loop.
});
const runtime = new RemoteNodeRuntime({
nodeConfig: createNode(),
projectId: "proj_7",
projectName: "Project 7",
});
(runtime as unknown as { reconnectBaseDelayMs: number }).reconnectBaseDelayMs = 1;
(runtime as unknown as { maxReconnectDelayMs: number }).maxReconnectDelayMs = 1;
(runtime as unknown as { maxReconnectAttempts: number }).maxReconnectAttempts = 3;
await runtime.start();
await vi.waitFor(() => {
expect(runtime.getStatus()).toBe("errored");
});
expect(mockStreamEvents.mock.calls.length).toBeGreaterThanOrEqual(3);
await runtime.stop();
});
it("validates remote node config on start", async () => {
const runtime = new RemoteNodeRuntime({
nodeConfig: createNode({ type: "local", url: undefined, apiKey: undefined }),
projectId: "proj_8",
projectName: "Project 8",
});
await expect(runtime.start()).rejects.toThrow("requires a remote node configuration");
});
});

View File

@@ -0,0 +1,343 @@
import { EventEmitter } from "node:events";
import type { NodeConfig, Task, TaskStore } from "@fusion/core";
import type { Scheduler } from "../scheduler.js";
import type {
ProjectRuntime,
ProjectRuntimeEvents,
RuntimeMetrics,
RuntimeStatus,
} from "../project-runtime.js";
import { remoteNodeLog } from "../logger.js";
import { RemoteNodeClient, type RemoteNodeEvent } from "./remote-node-client.js";
export interface RemoteNodeRuntimeConfig {
nodeConfig: NodeConfig;
projectId: string;
projectName: string;
}
export class RemoteNodeRuntime
extends EventEmitter<ProjectRuntimeEvents>
implements ProjectRuntime
{
private status: RuntimeStatus = "stopped";
private client: RemoteNodeClient;
private healthInterval: ReturnType<typeof setInterval> | null = null;
private streamLoopAbortController: AbortController | null = null;
private streamLoopPromise: Promise<void> | null = null;
private lastSuccessfulMetricsAt: string;
private cachedMetrics: RuntimeMetrics;
// Kept as mutable fields for testability.
private reconnectBaseDelayMs = 5_000;
private maxReconnectDelayMs = 60_000;
private maxReconnectAttempts = 10;
constructor(private config: RemoteNodeRuntimeConfig) {
super();
this.setMaxListeners(100);
this.cachedMetrics = {
inFlightTasks: 0,
activeAgents: 0,
lastActivityAt: new Date().toISOString(),
};
this.lastSuccessfulMetricsAt = this.cachedMetrics.lastActivityAt;
this.client = new RemoteNodeClient({
baseUrl: config.nodeConfig.url ?? "",
apiKey: config.nodeConfig.apiKey ?? "",
});
remoteNodeLog.log(
`Created RemoteNodeRuntime for project ${config.projectId} on node ${config.nodeConfig.name}`
);
}
async start(): Promise<void> {
if (this.status !== "stopped") {
throw new Error(`Cannot start runtime: current status is ${this.status}`);
}
this.validateRemoteNodeConfig();
this.setStatus("starting");
try {
await this.client.health();
await this.refreshMetrics();
this.setStatus("active");
this.startHealthChecks();
this.startEventStreamLoop();
remoteNodeLog.log(
`RemoteNodeRuntime started for ${this.config.projectId} (${this.config.projectName})`
);
} catch (error) {
const err = this.toError(error);
this.setStatus("errored");
this.emitRuntimeError(err);
throw err;
}
}
async stop(): Promise<void> {
if (this.status === "stopped" || this.status === "stopping") {
return;
}
this.setStatus("stopping");
if (this.healthInterval) {
clearInterval(this.healthInterval);
this.healthInterval = null;
}
if (this.streamLoopAbortController) {
this.streamLoopAbortController.abort();
}
if (this.streamLoopPromise) {
try {
await this.streamLoopPromise;
} catch {
// Best-effort shutdown. Errors are already emitted via runtime events.
}
}
this.streamLoopAbortController = null;
this.streamLoopPromise = null;
this.setStatus("stopped");
remoteNodeLog.log(`RemoteNodeRuntime stopped for ${this.config.projectId}`);
}
getStatus(): RuntimeStatus {
return this.status;
}
getTaskStore(): TaskStore {
throw new Error(
"TaskStore not accessible for remote node runtime. Use the remote Fusion API directly."
);
}
getScheduler(): Scheduler {
throw new Error("Scheduler not accessible for remote node runtime.");
}
getMetrics(): RuntimeMetrics {
void this.refreshMetrics();
return { ...this.cachedMetrics };
}
private validateRemoteNodeConfig(): void {
if (this.config.nodeConfig.type !== "remote") {
throw new Error(
`RemoteNodeRuntime requires a remote node configuration (received: ${this.config.nodeConfig.type})`
);
}
if (!this.config.nodeConfig.url) {
throw new Error("Remote node runtime requires nodeConfig.url for remote nodes.");
}
if (!this.config.nodeConfig.apiKey) {
throw new Error("Remote node runtime requires nodeConfig.apiKey for authentication.");
}
}
private startHealthChecks(): void {
if (this.healthInterval) {
clearInterval(this.healthInterval);
}
this.healthInterval = setInterval(() => {
void this.client.health().catch((error) => {
this.emitRuntimeError(this.toError(error));
});
}, 30_000);
}
private startEventStreamLoop(): void {
if (this.streamLoopPromise) {
return;
}
this.streamLoopAbortController = new AbortController();
this.streamLoopPromise = this.runEventStreamLoop(this.streamLoopAbortController.signal)
.catch((error) => {
this.emitRuntimeError(this.toError(error));
})
.finally(() => {
this.streamLoopPromise = null;
});
}
private async runEventStreamLoop(signal: AbortSignal): Promise<void> {
let reconnectAttempts = 0;
while (!signal.aborted && !this.isShuttingDown()) {
let sawAnyEvent = false;
try {
for await (const event of this.client.streamEvents({ signal })) {
if (signal.aborted || this.isShuttingDown()) {
return;
}
sawAnyEvent = true;
this.forwardRemoteEvent(event);
}
if (signal.aborted || this.isShuttingDown()) {
return;
}
if (sawAnyEvent) {
reconnectAttempts = 0;
}
throw new Error("Remote event stream ended unexpectedly");
} catch (error) {
if (signal.aborted || this.isShuttingDown()) {
return;
}
reconnectAttempts += 1;
this.emitRuntimeError(this.toError(error));
if (reconnectAttempts >= this.maxReconnectAttempts) {
this.setStatus("errored");
return;
}
const delayMs = Math.min(
this.reconnectBaseDelayMs * 2 ** (reconnectAttempts - 1),
this.maxReconnectDelayMs
);
remoteNodeLog.warn(
`Remote event stream disconnected for ${this.config.projectId}; reconnecting in ${delayMs}ms ` +
`(attempt ${reconnectAttempts}/${this.maxReconnectAttempts})`
);
await this.sleep(delayMs, signal);
if (signal.aborted) {
return;
}
try {
await this.client.health();
} catch (healthError) {
this.emitRuntimeError(this.toError(healthError));
}
}
}
}
private forwardRemoteEvent(event: RemoteNodeEvent): void {
switch (event.type) {
case "task:created":
this.emit("task:created", event.payload as Task);
break;
case "task:moved": {
const payload = event.payload as { task: Task; from: string; to: string };
this.emit("task:moved", {
task: payload.task,
from: payload.from,
to: payload.to,
});
break;
}
case "task:updated":
this.emit("task:updated", event.payload as Task);
break;
case "error": {
const payload = event.payload;
if (payload instanceof Error) {
this.emitRuntimeError(payload);
} else if (typeof payload === "object" && payload && "message" in payload) {
const message = String((payload as { message: unknown }).message);
this.emitRuntimeError(new Error(message));
} else {
this.emitRuntimeError(new Error(String(payload)));
}
break;
}
default:
remoteNodeLog.warn(
`Ignoring unsupported remote event type "${event.type}" for ${this.config.projectId}`
);
}
}
private async refreshMetrics(): Promise<RuntimeMetrics> {
try {
const metrics = await this.client.getMetrics();
this.cachedMetrics = { ...metrics };
this.lastSuccessfulMetricsAt = metrics.lastActivityAt;
return metrics;
} catch {
const fallback: RuntimeMetrics = {
inFlightTasks: 0,
activeAgents: 0,
lastActivityAt: this.lastSuccessfulMetricsAt,
};
this.cachedMetrics = fallback;
return fallback;
}
}
private setStatus(newStatus: RuntimeStatus): void {
const previous = this.status;
this.status = newStatus;
if (previous !== newStatus) {
this.emit("health-changed", {
status: newStatus,
previous,
});
}
}
private emitRuntimeError(error: Error): void {
if (this.listenerCount("error") > 0) {
this.emit("error", error);
return;
}
remoteNodeLog.error(
`Unhandled remote runtime error for ${this.config.projectId}: ${error.message}`
);
}
private isShuttingDown(): boolean {
return this.status === "stopping" || this.status === "stopped";
}
private toError(error: unknown): Error {
return error instanceof Error ? error : new Error(String(error));
}
private async sleep(ms: number, signal?: AbortSignal): Promise<void> {
await new Promise<void>((resolve) => {
const timeout = setTimeout(() => {
cleanup();
resolve();
}, ms);
const onAbort = () => {
cleanup();
resolve();
};
const cleanup = () => {
clearTimeout(timeout);
signal?.removeEventListener("abort", onAbort);
};
signal?.addEventListener("abort", onAbort, { once: true });
});
}
}