feat(FN-1079): add remote node runtime orchestration
- Add RemoteNodeClient and RemoteNodeRuntime with comprehensive lifecycle, status, and metrics test coverage - Route projects by node assignment in ProjectManager and integrate remote runtime handling in HybridExecutor - Introduce NodeHealthMonitor and wire diagnostic logging/export updates for remote node health tracking - Harden remote runtime shutdown checks and type-safety paths surfaced during review feedback
This commit is contained in:
@@ -6,64 +6,106 @@ import type { ProjectRuntimeConfig } from "../project-runtime.js";
|
||||
// Mock the ProjectManager
|
||||
const mockRuntimes = new Map();
|
||||
const mockProjectIds: string[] = [];
|
||||
const mockProjectManagerInstances: Array<{
|
||||
addProject: ReturnType<typeof vi.fn>;
|
||||
removeProject: ReturnType<typeof vi.fn>;
|
||||
getRuntime: ReturnType<typeof vi.fn>;
|
||||
listRuntimes: ReturnType<typeof vi.fn>;
|
||||
getProjectIds: ReturnType<typeof vi.fn>;
|
||||
getGlobalMetrics: ReturnType<typeof vi.fn>;
|
||||
acquireGlobalSlot: ReturnType<typeof vi.fn>;
|
||||
releaseGlobalSlot: ReturnType<typeof vi.fn>;
|
||||
stopAll: ReturnType<typeof vi.fn>;
|
||||
on: ReturnType<typeof vi.fn>;
|
||||
}> = [];
|
||||
|
||||
vi.mock("../project-manager.js", () => ({
|
||||
ProjectManager: vi.fn().mockImplementation(() => ({
|
||||
addProject: vi.fn().mockImplementation((config: ProjectRuntimeConfig) => {
|
||||
const runtime = {
|
||||
start: vi.fn().mockResolvedValue(undefined),
|
||||
stop: vi.fn().mockResolvedValue(undefined),
|
||||
getStatus: vi.fn().mockReturnValue("active"),
|
||||
getTaskStore: vi.fn(),
|
||||
getScheduler: vi.fn(),
|
||||
getMetrics: vi.fn().mockReturnValue({
|
||||
inFlightTasks: 0,
|
||||
activeAgents: 0,
|
||||
lastActivityAt: new Date().toISOString(),
|
||||
}),
|
||||
on: vi.fn().mockReturnThis(),
|
||||
};
|
||||
mockRuntimes.set(config.projectId, runtime);
|
||||
mockProjectIds.push(config.projectId);
|
||||
return Promise.resolve(runtime);
|
||||
}),
|
||||
removeProject: vi.fn().mockImplementation((projectId: string) => {
|
||||
mockRuntimes.delete(projectId);
|
||||
const index = mockProjectIds.indexOf(projectId);
|
||||
if (index > -1) mockProjectIds.splice(index, 1);
|
||||
return Promise.resolve(undefined);
|
||||
}),
|
||||
getRuntime: vi.fn().mockImplementation((projectId: string) => {
|
||||
return mockRuntimes.get(projectId);
|
||||
}),
|
||||
listRuntimes: vi.fn().mockImplementation(() => {
|
||||
return Array.from(mockRuntimes.values());
|
||||
}),
|
||||
getProjectIds: vi.fn().mockImplementation(() => {
|
||||
return [...mockProjectIds];
|
||||
}),
|
||||
getGlobalMetrics: vi.fn().mockResolvedValue({
|
||||
totalInFlightTasks: 0,
|
||||
totalActiveAgents: 0,
|
||||
runtimeCountByStatus: {
|
||||
active: 0,
|
||||
paused: 0,
|
||||
errored: 0,
|
||||
stopped: 0,
|
||||
starting: 0,
|
||||
stopping: 0,
|
||||
},
|
||||
totalRuntimes: 0,
|
||||
}),
|
||||
acquireGlobalSlot: vi.fn().mockResolvedValue(true),
|
||||
releaseGlobalSlot: vi.fn().mockResolvedValue(undefined),
|
||||
stopAll: vi.fn().mockImplementation(() => {
|
||||
mockRuntimes.clear();
|
||||
mockProjectIds.length = 0;
|
||||
return Promise.resolve(undefined);
|
||||
}),
|
||||
on: vi.fn().mockReturnThis(),
|
||||
})),
|
||||
ProjectManager: vi.fn().mockImplementation(() => {
|
||||
const instance = {
|
||||
addProject: vi.fn().mockImplementation((config: ProjectRuntimeConfig) => {
|
||||
const runtime = {
|
||||
start: vi.fn().mockResolvedValue(undefined),
|
||||
stop: vi.fn().mockResolvedValue(undefined),
|
||||
getStatus: vi.fn().mockReturnValue("active"),
|
||||
getTaskStore: vi.fn(),
|
||||
getScheduler: vi.fn(),
|
||||
getMetrics: vi.fn().mockReturnValue({
|
||||
inFlightTasks: 0,
|
||||
activeAgents: 0,
|
||||
lastActivityAt: new Date().toISOString(),
|
||||
}),
|
||||
on: vi.fn().mockReturnThis(),
|
||||
};
|
||||
mockRuntimes.set(config.projectId, runtime);
|
||||
mockProjectIds.push(config.projectId);
|
||||
return Promise.resolve(runtime);
|
||||
}),
|
||||
removeProject: vi.fn().mockImplementation((projectId: string) => {
|
||||
mockRuntimes.delete(projectId);
|
||||
const index = mockProjectIds.indexOf(projectId);
|
||||
if (index > -1) mockProjectIds.splice(index, 1);
|
||||
return Promise.resolve(undefined);
|
||||
}),
|
||||
getRuntime: vi.fn().mockImplementation((projectId: string) => {
|
||||
return mockRuntimes.get(projectId);
|
||||
}),
|
||||
listRuntimes: vi.fn().mockImplementation(() => {
|
||||
return Array.from(mockRuntimes.values());
|
||||
}),
|
||||
getProjectIds: vi.fn().mockImplementation(() => {
|
||||
return [...mockProjectIds];
|
||||
}),
|
||||
getGlobalMetrics: vi.fn().mockResolvedValue({
|
||||
totalInFlightTasks: 0,
|
||||
totalActiveAgents: 0,
|
||||
runtimeCountByStatus: {
|
||||
active: 0,
|
||||
paused: 0,
|
||||
errored: 0,
|
||||
stopped: 0,
|
||||
starting: 0,
|
||||
stopping: 0,
|
||||
},
|
||||
totalRuntimes: 0,
|
||||
}),
|
||||
acquireGlobalSlot: vi.fn().mockResolvedValue(true),
|
||||
releaseGlobalSlot: vi.fn().mockResolvedValue(undefined),
|
||||
stopAll: vi.fn().mockImplementation(() => {
|
||||
mockRuntimes.clear();
|
||||
mockProjectIds.length = 0;
|
||||
return Promise.resolve(undefined);
|
||||
}),
|
||||
on: vi.fn().mockReturnThis(),
|
||||
};
|
||||
|
||||
mockProjectManagerInstances.push(instance);
|
||||
return instance;
|
||||
}),
|
||||
}));
|
||||
|
||||
const mockNodeHealthMonitorInstances: Array<{
|
||||
start: ReturnType<typeof vi.fn>;
|
||||
stop: ReturnType<typeof vi.fn>;
|
||||
checkAllNodes: ReturnType<typeof vi.fn>;
|
||||
}> = [];
|
||||
|
||||
vi.mock("../node-health-monitor.js", () => ({
|
||||
NodeHealthMonitor: vi.fn().mockImplementation(() => {
|
||||
const instance = {
|
||||
start: vi.fn().mockResolvedValue(undefined),
|
||||
stop: vi.fn().mockResolvedValue(undefined),
|
||||
checkAllNodes: vi.fn().mockResolvedValue({
|
||||
checked: 0,
|
||||
online: 0,
|
||||
offline: 0,
|
||||
error: 0,
|
||||
connecting: 0,
|
||||
}),
|
||||
};
|
||||
|
||||
mockNodeHealthMonitorInstances.push(instance);
|
||||
return instance;
|
||||
}),
|
||||
}));
|
||||
|
||||
describe("HybridExecutor", () => {
|
||||
@@ -112,6 +154,8 @@ describe("HybridExecutor", () => {
|
||||
vi.clearAllMocks();
|
||||
mockRuntimes.clear();
|
||||
mockProjectIds.length = 0;
|
||||
mockProjectManagerInstances.length = 0;
|
||||
mockNodeHealthMonitorInstances.length = 0;
|
||||
});
|
||||
|
||||
describe("initialization", () => {
|
||||
@@ -129,6 +173,30 @@ describe("HybridExecutor", () => {
|
||||
expect(mockCentralCore.listProjects).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should forward remote-node assigned projects to ProjectManager for routing", async () => {
|
||||
const remoteAssignedProject: RegisteredProject = {
|
||||
...mockProject,
|
||||
id: "proj_remote_1",
|
||||
name: "Remote Assigned Project",
|
||||
status: "active",
|
||||
nodeId: "node_remote_1",
|
||||
};
|
||||
(mockCentralCore.listProjects as ReturnType<typeof vi.fn>).mockResolvedValue([
|
||||
remoteAssignedProject,
|
||||
]);
|
||||
|
||||
await executor.initialize();
|
||||
|
||||
const manager = mockProjectManagerInstances[0];
|
||||
expect(manager?.addProject).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
projectId: "proj_remote_1",
|
||||
isolationMode: "in-process",
|
||||
workingDirectory: "/tmp/test-project",
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("should setup CentralCore listeners on initialize", async () => {
|
||||
await executor.initialize();
|
||||
expect(mockCentralCore.on).toHaveBeenCalledWith(
|
||||
@@ -145,12 +213,22 @@ describe("HybridExecutor", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("should create and start node health monitor on initialize", async () => {
|
||||
await executor.initialize();
|
||||
|
||||
expect(mockNodeHealthMonitorInstances).toHaveLength(1);
|
||||
expect(mockNodeHealthMonitorInstances[0]?.start).toHaveBeenCalledTimes(1);
|
||||
expect(executor.getNodeHealthMonitor()).toBe(mockNodeHealthMonitorInstances[0]);
|
||||
});
|
||||
|
||||
it("should be idempotent (initialize multiple times)", async () => {
|
||||
await executor.initialize();
|
||||
const callCount = (mockCentralCore.listProjects as ReturnType<typeof vi.fn>).mock.calls.length;
|
||||
|
||||
await executor.initialize();
|
||||
expect(mockCentralCore.listProjects).toHaveBeenCalledTimes(callCount);
|
||||
expect(mockNodeHealthMonitorInstances).toHaveLength(1);
|
||||
expect(mockNodeHealthMonitorInstances[0]?.start).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -319,6 +397,21 @@ describe("HybridExecutor", () => {
|
||||
"project:updated"
|
||||
);
|
||||
});
|
||||
|
||||
it("should stop node health monitor before stopping runtimes", async () => {
|
||||
const monitor = mockNodeHealthMonitorInstances[0];
|
||||
const manager = mockProjectManagerInstances[0];
|
||||
|
||||
await executor.shutdown();
|
||||
|
||||
expect(monitor?.stop).toHaveBeenCalledTimes(1);
|
||||
expect(manager?.stopAll).toHaveBeenCalledTimes(1);
|
||||
|
||||
const monitorStopOrder = monitor?.stop.mock.invocationCallOrder[0] ?? Number.MAX_SAFE_INTEGER;
|
||||
const stopAllOrder = manager?.stopAll.mock.invocationCallOrder[0] ?? Number.MIN_SAFE_INTEGER;
|
||||
expect(monitorStopOrder).toBeLessThan(stopAllOrder);
|
||||
expect(executor.getNodeHealthMonitor()).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("updateProject", () => {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import type { Task, CentralCore, RegisteredProject } from "@fusion/core";
|
||||
import { ProjectManager } from "./project-manager.js";
|
||||
import { NodeHealthMonitor } from "./node-health-monitor.js";
|
||||
import type {
|
||||
ProjectRuntime,
|
||||
ProjectRuntimeConfig,
|
||||
@@ -133,6 +134,7 @@ export interface HybridExecutorOptions {
|
||||
*/
|
||||
export class HybridExecutor extends EventEmitter<HybridExecutorEvents> {
|
||||
private projectManager: ProjectManager;
|
||||
private nodeHealthMonitor: NodeHealthMonitor | null = null;
|
||||
private initialized = false;
|
||||
|
||||
/**
|
||||
@@ -197,6 +199,10 @@ export class HybridExecutor extends EventEmitter<HybridExecutorEvents> {
|
||||
// Listen for CentralCore project events
|
||||
this.setupCentralCoreListeners();
|
||||
|
||||
// Start remote node health monitoring after project runtimes are loaded.
|
||||
this.nodeHealthMonitor = new NodeHealthMonitor(this.centralCore);
|
||||
await this.nodeHealthMonitor.start();
|
||||
|
||||
this.initialized = true;
|
||||
hybridExecutorLog.log("HybridExecutor initialized");
|
||||
}
|
||||
@@ -318,6 +324,13 @@ export class HybridExecutor extends EventEmitter<HybridExecutorEvents> {
|
||||
return this.projectManager.getGlobalMetrics();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the optional node health monitor instance.
|
||||
*/
|
||||
getNodeHealthMonitor(): NodeHealthMonitor | null {
|
||||
return this.nodeHealthMonitor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire a global concurrency slot.
|
||||
*
|
||||
@@ -355,6 +368,12 @@ export class HybridExecutor extends EventEmitter<HybridExecutorEvents> {
|
||||
this.centralCore.removeAllListeners("project:unregistered");
|
||||
this.centralCore.removeAllListeners("project:updated");
|
||||
|
||||
// Stop node health monitor before shutting down runtimes.
|
||||
if (this.nodeHealthMonitor) {
|
||||
await this.nodeHealthMonitor.stop();
|
||||
this.nodeHealthMonitor = null;
|
||||
}
|
||||
|
||||
// Stop all runtimes
|
||||
await this.projectManager.stopAll();
|
||||
|
||||
|
||||
@@ -19,6 +19,9 @@ export { StuckTaskDetector, type StuckTaskDetectorOptions, type DisposableSessio
|
||||
export { TokenCapDetector, type TokenCapCheckResult } from "./token-cap-detector.js";
|
||||
export { SelfHealingManager, type SelfHealingOptions } from "./self-healing.js";
|
||||
export { ProjectManager } from "./project-manager.js";
|
||||
export { NodeHealthMonitor } from "./node-health-monitor.js";
|
||||
export { RemoteNodeClient } from "./runtimes/remote-node-client.js";
|
||||
export { RemoteNodeRuntime, type RemoteNodeRuntimeConfig } from "./runtimes/remote-node-runtime.js";
|
||||
export { StepSessionExecutor } from "./step-session-executor.js";
|
||||
export type { StepResult, ParallelWave, StepSessionExecutorOptions } from "./step-session-executor.js";
|
||||
// Multi-project runtime types
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { createLogger, schedulerLog, executorLog, triageLog, mergerLog, worktreePoolLog, reviewerLog } from "./logger.js";
|
||||
import {
|
||||
createLogger,
|
||||
schedulerLog,
|
||||
executorLog,
|
||||
triageLog,
|
||||
mergerLog,
|
||||
worktreePoolLog,
|
||||
reviewerLog,
|
||||
remoteNodeLog,
|
||||
} from "./logger.js";
|
||||
|
||||
describe("createLogger", () => {
|
||||
let logSpy: ReturnType<typeof vi.spyOn>;
|
||||
@@ -71,5 +80,8 @@ describe("createLogger", () => {
|
||||
|
||||
reviewerLog.log("review");
|
||||
expect(logSpy).toHaveBeenCalledWith("[reviewer] review");
|
||||
|
||||
remoteNodeLog.log("stream");
|
||||
expect(logSpy).toHaveBeenCalledWith("[remote-node] stream");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -82,3 +82,9 @@ export const autopilotLog = createLogger("autopilot");
|
||||
|
||||
/** Logger for the heartbeat execution subsystem. */
|
||||
export const heartbeatLog = createLogger("heartbeat");
|
||||
|
||||
/** Logger for remote node runtime/client subsystems. */
|
||||
export const remoteNodeLog = createLogger("remote-node");
|
||||
|
||||
/** Logger for periodic node health monitor subsystem. */
|
||||
export const nodeHealthMonitorLog = createLogger("node-health-monitor");
|
||||
|
||||
137
packages/engine/src/node-health-monitor.test.ts
Normal file
137
packages/engine/src/node-health-monitor.test.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CentralCore, NodeConfig } from "@fusion/core";
|
||||
import { NodeHealthMonitor } from "./node-health-monitor.js";
|
||||
|
||||
const NOW = "2026-04-08T00:00:00.000Z";
|
||||
|
||||
function createNode(overrides: Partial<NodeConfig>): NodeConfig {
|
||||
return {
|
||||
id: "node-id",
|
||||
name: "Node",
|
||||
type: "remote",
|
||||
status: "online",
|
||||
maxConcurrent: 4,
|
||||
createdAt: NOW,
|
||||
updatedAt: NOW,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("NodeHealthMonitor", () => {
|
||||
let mockCentralCore: CentralCore;
|
||||
let listNodesMock: ReturnType<typeof vi.fn>;
|
||||
let checkNodeHealthMock: ReturnType<typeof vi.fn>;
|
||||
let monitor: NodeHealthMonitor;
|
||||
let warnSpy: ReturnType<typeof vi.spyOn>;
|
||||
let logSpy: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
|
||||
listNodesMock = vi.fn().mockResolvedValue([
|
||||
createNode({ id: "node-local", name: "Local Node", type: "local", status: "online" }),
|
||||
createNode({ id: "node-remote", name: "Remote Node", type: "remote", status: "online" }),
|
||||
]);
|
||||
checkNodeHealthMock = vi.fn().mockResolvedValue("online");
|
||||
|
||||
mockCentralCore = {
|
||||
listNodes: listNodesMock,
|
||||
checkNodeHealth: checkNodeHealthMock,
|
||||
} as unknown as CentralCore;
|
||||
|
||||
monitor = new NodeHealthMonitor(mockCentralCore, { checkIntervalMs: 1_000 });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await monitor.stop();
|
||||
vi.useRealTimers();
|
||||
warnSpy.mockRestore();
|
||||
logSpy.mockRestore();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("start() sets interval and checks only remote nodes", async () => {
|
||||
await monitor.start();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
|
||||
expect(listNodesMock).toHaveBeenCalled();
|
||||
expect(checkNodeHealthMock).toHaveBeenCalledTimes(1);
|
||||
expect(checkNodeHealthMock).toHaveBeenCalledWith("node-remote");
|
||||
});
|
||||
|
||||
it("stop() clears interval", async () => {
|
||||
await monitor.start();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
expect(checkNodeHealthMock).toHaveBeenCalledTimes(1);
|
||||
|
||||
await monitor.stop();
|
||||
checkNodeHealthMock.mockClear();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2_000);
|
||||
expect(checkNodeHealthMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("checkAllNodes() checks each remote node and skips local nodes", async () => {
|
||||
await monitor.start();
|
||||
|
||||
const summary = await monitor.checkAllNodes();
|
||||
|
||||
expect(checkNodeHealthMock).toHaveBeenCalledTimes(1);
|
||||
expect(checkNodeHealthMock).toHaveBeenCalledWith("node-remote");
|
||||
expect(summary).toEqual({
|
||||
checked: 1,
|
||||
online: 1,
|
||||
offline: 0,
|
||||
error: 0,
|
||||
connecting: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("logs warning when node transitions from online to offline", async () => {
|
||||
checkNodeHealthMock.mockResolvedValue("offline");
|
||||
|
||||
await monitor.start();
|
||||
await monitor.checkAllNodes();
|
||||
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[node-health-monitor] Remote node Remote Node (node-remote) degraded")
|
||||
);
|
||||
expect(monitor.getNodeHealth("node-remote")).toBe("offline");
|
||||
});
|
||||
|
||||
it("logs recovery when node transitions back to online", async () => {
|
||||
checkNodeHealthMock.mockResolvedValueOnce("offline").mockResolvedValueOnce("online");
|
||||
|
||||
await monitor.start();
|
||||
await monitor.checkAllNodes();
|
||||
await monitor.checkAllNodes();
|
||||
|
||||
expect(logSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[node-health-monitor] Remote node Remote Node (node-remote) recovered")
|
||||
);
|
||||
expect(monitor.getNodeHealth("node-remote")).toBe("online");
|
||||
});
|
||||
|
||||
it("is a no-op when no remote nodes are registered", async () => {
|
||||
listNodesMock.mockResolvedValue([
|
||||
createNode({ id: "node-local-only", name: "Local Only", type: "local", status: "online" }),
|
||||
]);
|
||||
|
||||
await monitor.start();
|
||||
const summary = await monitor.checkAllNodes();
|
||||
|
||||
expect(checkNodeHealthMock).not.toHaveBeenCalled();
|
||||
expect(summary).toEqual({
|
||||
checked: 0,
|
||||
online: 0,
|
||||
offline: 0,
|
||||
error: 0,
|
||||
connecting: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
153
packages/engine/src/node-health-monitor.ts
Normal file
153
packages/engine/src/node-health-monitor.ts
Normal file
@@ -0,0 +1,153 @@
|
||||
import type { CentralCore, NodeStatus } from "@fusion/core";
|
||||
import { nodeHealthMonitorLog } from "./logger.js";
|
||||
|
||||
export interface NodeHealthMonitorOptions {
|
||||
checkIntervalMs?: number;
|
||||
}
|
||||
|
||||
export interface NodeHealthCheckSummary {
|
||||
checked: number;
|
||||
online: number;
|
||||
offline: number;
|
||||
error: number;
|
||||
connecting: number;
|
||||
}
|
||||
|
||||
export class NodeHealthMonitor {
|
||||
private readonly checkIntervalMs: number;
|
||||
private interval: ReturnType<typeof setInterval> | null = null;
|
||||
private running = false;
|
||||
private lastKnownStatus = new Map<string, NodeStatus>();
|
||||
private activeCheck: Promise<NodeHealthCheckSummary> | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly centralCore: CentralCore,
|
||||
options: NodeHealthMonitorOptions = {}
|
||||
) {
|
||||
this.checkIntervalMs = options.checkIntervalMs ?? 60_000;
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
if (this.running) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.running = true;
|
||||
|
||||
const nodes = await this.centralCore.listNodes();
|
||||
for (const node of nodes) {
|
||||
if (node.type === "remote") {
|
||||
this.lastKnownStatus.set(node.id, node.status);
|
||||
}
|
||||
}
|
||||
|
||||
this.interval = setInterval(() => {
|
||||
void this.checkAllNodes();
|
||||
}, this.checkIntervalMs);
|
||||
|
||||
nodeHealthMonitorLog.log(
|
||||
`NodeHealthMonitor started (${this.lastKnownStatus.size} remote nodes, interval=${this.checkIntervalMs}ms)`
|
||||
);
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.running = false;
|
||||
|
||||
if (this.interval) {
|
||||
clearInterval(this.interval);
|
||||
this.interval = null;
|
||||
}
|
||||
|
||||
if (this.activeCheck) {
|
||||
await this.activeCheck.catch(() => {
|
||||
// Best-effort: pending check errors are already logged.
|
||||
});
|
||||
}
|
||||
|
||||
nodeHealthMonitorLog.log("NodeHealthMonitor stopped");
|
||||
}
|
||||
|
||||
async checkAllNodes(): Promise<NodeHealthCheckSummary> {
|
||||
if (!this.running) {
|
||||
return {
|
||||
checked: 0,
|
||||
online: 0,
|
||||
offline: 0,
|
||||
error: 0,
|
||||
connecting: 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (this.activeCheck) {
|
||||
return this.activeCheck;
|
||||
}
|
||||
|
||||
this.activeCheck = this.runCheckAllNodes();
|
||||
try {
|
||||
return await this.activeCheck;
|
||||
} finally {
|
||||
this.activeCheck = null;
|
||||
}
|
||||
}
|
||||
|
||||
getNodeHealth(nodeId: string): NodeStatus | undefined {
|
||||
return this.lastKnownStatus.get(nodeId);
|
||||
}
|
||||
|
||||
private async runCheckAllNodes(): Promise<NodeHealthCheckSummary> {
|
||||
const nodes = await this.centralCore.listNodes();
|
||||
const remoteNodes = nodes.filter((node) => node.type === "remote");
|
||||
|
||||
if (remoteNodes.length === 0) {
|
||||
return {
|
||||
checked: 0,
|
||||
online: 0,
|
||||
offline: 0,
|
||||
error: 0,
|
||||
connecting: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const summary: NodeHealthCheckSummary = {
|
||||
checked: 0,
|
||||
online: 0,
|
||||
offline: 0,
|
||||
error: 0,
|
||||
connecting: 0,
|
||||
};
|
||||
|
||||
for (const node of remoteNodes) {
|
||||
try {
|
||||
const previousStatus = this.lastKnownStatus.get(node.id) ?? node.status;
|
||||
const nextStatus = await this.centralCore.checkNodeHealth(node.id);
|
||||
|
||||
this.lastKnownStatus.set(node.id, nextStatus);
|
||||
summary.checked += 1;
|
||||
summary[nextStatus] += 1;
|
||||
|
||||
if (previousStatus !== nextStatus) {
|
||||
if (previousStatus === "online" && (nextStatus === "offline" || nextStatus === "error")) {
|
||||
nodeHealthMonitorLog.warn(
|
||||
`Remote node ${node.name} (${node.id}) degraded: ${previousStatus} → ${nextStatus}`
|
||||
);
|
||||
} else if (previousStatus !== "online" && nextStatus === "online") {
|
||||
nodeHealthMonitorLog.log(
|
||||
`Remote node ${node.name} (${node.id}) recovered: ${previousStatus} → online`
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.lastKnownStatus.set(node.id, "error");
|
||||
summary.checked += 1;
|
||||
summary.error += 1;
|
||||
nodeHealthMonitorLog.warn(
|
||||
`Failed to check node ${node.name} (${node.id}) health: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,8 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import type { CentralCore, RegisteredProject, Task } from "@fusion/core";
|
||||
import { InProcessRuntime } from "./runtimes/in-process-runtime.js";
|
||||
import { ChildProcessRuntime } from "./runtimes/child-process-runtime.js";
|
||||
import { RemoteNodeRuntime } from "./runtimes/remote-node-runtime.js";
|
||||
import { ProjectManager } from "./project-manager.js";
|
||||
import type { ProjectRuntimeConfig } from "./project-runtime.js";
|
||||
|
||||
@@ -40,6 +43,26 @@ vi.mock("./runtimes/child-process-runtime.js", () => ({
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("./runtimes/remote-node-runtime.js", () => ({
|
||||
RemoteNodeRuntime: vi.fn().mockImplementation(() => ({
|
||||
start: vi.fn().mockResolvedValue(undefined),
|
||||
stop: vi.fn().mockResolvedValue(undefined),
|
||||
getStatus: vi.fn().mockReturnValue("active"),
|
||||
getTaskStore: vi.fn().mockImplementation(() => {
|
||||
throw new Error("TaskStore not accessible for remote node runtime");
|
||||
}),
|
||||
getScheduler: vi.fn().mockImplementation(() => {
|
||||
throw new Error("Scheduler not accessible for remote node runtime");
|
||||
}),
|
||||
getMetrics: vi.fn().mockReturnValue({
|
||||
inFlightTasks: 0,
|
||||
activeAgents: 0,
|
||||
lastActivityAt: new Date().toISOString(),
|
||||
}),
|
||||
on: vi.fn().mockReturnThis(),
|
||||
})),
|
||||
}));
|
||||
|
||||
describe("ProjectManager", () => {
|
||||
let manager: ProjectManager;
|
||||
let mockCentralCore: CentralCore;
|
||||
@@ -56,6 +79,7 @@ describe("ProjectManager", () => {
|
||||
beforeEach(() => {
|
||||
mockCentralCore = {
|
||||
getProject: vi.fn().mockResolvedValue(mockProject),
|
||||
getNode: vi.fn().mockResolvedValue(undefined),
|
||||
getGlobalConcurrencyState: vi.fn().mockResolvedValue({
|
||||
globalMaxConcurrent: 4,
|
||||
currentlyActive: 0,
|
||||
@@ -143,6 +167,84 @@ describe("ProjectManager", () => {
|
||||
{ status: "active" }
|
||||
);
|
||||
});
|
||||
|
||||
it("routes to RemoteNodeRuntime when assigned node is remote", async () => {
|
||||
(mockCentralCore.getProject as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
...mockProject,
|
||||
nodeId: "node_remote_1",
|
||||
});
|
||||
(mockCentralCore.getNode as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
id: "node_remote_1",
|
||||
name: "Remote 1",
|
||||
type: "remote",
|
||||
url: "https://remote.example.com",
|
||||
apiKey: "remote-token",
|
||||
status: "online",
|
||||
maxConcurrent: 4,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
await manager.addProject(testConfig);
|
||||
|
||||
expect(RemoteNodeRuntime).toHaveBeenCalledWith({
|
||||
nodeConfig: expect.objectContaining({ id: "node_remote_1", type: "remote" }),
|
||||
projectId: "proj_test123",
|
||||
projectName: "Test Project",
|
||||
});
|
||||
});
|
||||
|
||||
it("routes to InProcessRuntime when assigned node is local", async () => {
|
||||
(mockCentralCore.getProject as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
...mockProject,
|
||||
nodeId: "node_local_1",
|
||||
});
|
||||
(mockCentralCore.getNode as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
id: "node_local_1",
|
||||
name: "Local 1",
|
||||
type: "local",
|
||||
status: "online",
|
||||
maxConcurrent: 4,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
await manager.addProject(testConfig);
|
||||
|
||||
expect(InProcessRuntime).toHaveBeenCalled();
|
||||
expect(RemoteNodeRuntime).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("routes to InProcessRuntime when no node assignment exists", async () => {
|
||||
(mockCentralCore.getProject as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
...mockProject,
|
||||
nodeId: undefined,
|
||||
});
|
||||
|
||||
await manager.addProject(testConfig);
|
||||
|
||||
expect(InProcessRuntime).toHaveBeenCalled();
|
||||
expect(mockCentralCore.getNode).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("falls back to InProcessRuntime and logs warning when assigned node is missing", async () => {
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
(mockCentralCore.getProject as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
...mockProject,
|
||||
nodeId: "node_missing",
|
||||
});
|
||||
(mockCentralCore.getNode as ReturnType<typeof vi.fn>).mockResolvedValue(undefined);
|
||||
|
||||
await manager.addProject(testConfig);
|
||||
|
||||
expect(InProcessRuntime).toHaveBeenCalled();
|
||||
expect(RemoteNodeRuntime).not.toHaveBeenCalled();
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[project-manager] Assigned node node_missing not found")
|
||||
);
|
||||
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe("removeProject", () => {
|
||||
|
||||
@@ -2,6 +2,7 @@ import { EventEmitter } from "node:events";
|
||||
import type { Task, CentralCore, RegisteredProject } from "@fusion/core";
|
||||
import { InProcessRuntime } from "./runtimes/in-process-runtime.js";
|
||||
import { ChildProcessRuntime } from "./runtimes/child-process-runtime.js";
|
||||
import { RemoteNodeRuntime } from "./runtimes/remote-node-runtime.js";
|
||||
import { AgentSemaphore } from "./concurrency.js";
|
||||
import type {
|
||||
ProjectRuntime,
|
||||
@@ -163,8 +164,28 @@ export class ProjectManager extends EventEmitter<ProjectManagerEvents> {
|
||||
if (config.isolationMode === "child-process") {
|
||||
runtime = new ChildProcessRuntime(config, this.centralCore);
|
||||
} else {
|
||||
// Default to in-process
|
||||
runtime = new InProcessRuntime(config, this.centralCore);
|
||||
let assignedNode = undefined;
|
||||
|
||||
if (project.nodeId) {
|
||||
assignedNode = await this.centralCore.getNode(project.nodeId);
|
||||
|
||||
if (!assignedNode) {
|
||||
projectManagerLog.warn(
|
||||
`Assigned node ${project.nodeId} not found for project ${project.id}; falling back to InProcessRuntime`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (assignedNode?.type === "remote") {
|
||||
runtime = new RemoteNodeRuntime({
|
||||
nodeConfig: assignedNode,
|
||||
projectId: config.projectId,
|
||||
projectName: project.name,
|
||||
});
|
||||
} else {
|
||||
// Default to local in-process runtime (includes unassigned + local-node assigned)
|
||||
runtime = new InProcessRuntime(config, this.centralCore);
|
||||
}
|
||||
}
|
||||
|
||||
// Set up event forwarding with project attribution
|
||||
|
||||
323
packages/engine/src/runtimes/remote-node-client.test.ts
Normal file
323
packages/engine/src/runtimes/remote-node-client.test.ts
Normal file
@@ -0,0 +1,323 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { RuntimeMetrics } from "../project-runtime.js";
|
||||
import { RemoteNodeClient } from "./remote-node-client.js";
|
||||
|
||||
const BASE_URL = "https://node.example.com";
|
||||
const API_KEY = "secret-token";
|
||||
|
||||
describe("RemoteNodeClient", () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("health() parses successful response and sends auth header", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ status: "ok", version: "1.0.0", uptime: 123 }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
|
||||
const health = await client.health();
|
||||
|
||||
expect(health).toEqual({ status: "ok", version: "1.0.0", uptime: 123 });
|
||||
expect(fetchMock).toHaveBeenCalledWith(`${BASE_URL}/api/health`, expect.objectContaining({
|
||||
method: "GET",
|
||||
headers: expect.objectContaining({
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
}),
|
||||
}));
|
||||
});
|
||||
|
||||
it("getMetrics() parses runtime metrics", async () => {
|
||||
const metrics: RuntimeMetrics = {
|
||||
inFlightTasks: 4,
|
||||
activeAgents: 2,
|
||||
lastActivityAt: "2026-04-08T00:00:00.000Z",
|
||||
};
|
||||
|
||||
globalThis.fetch = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify(metrics), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
|
||||
await expect(client.getMetrics()).resolves.toEqual(metrics);
|
||||
});
|
||||
|
||||
it("createTask() sends POST with JSON body", async () => {
|
||||
const createdTask = {
|
||||
id: "KB-001",
|
||||
description: "Create me",
|
||||
column: "todo",
|
||||
dependencies: [],
|
||||
steps: [],
|
||||
currentStep: 0,
|
||||
status: "pending",
|
||||
log: [],
|
||||
attachments: [],
|
||||
createdAt: "2026-04-08T00:00:00.000Z",
|
||||
updatedAt: "2026-04-08T00:00:00.000Z",
|
||||
size: "M",
|
||||
reviewLevel: 1,
|
||||
};
|
||||
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify(createdTask), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
|
||||
await client.createTask({ description: "Create me" });
|
||||
|
||||
const options = fetchMock.mock.calls[0]?.[1] as RequestInit;
|
||||
expect(fetchMock).toHaveBeenCalledWith(`${BASE_URL}/api/tasks`, expect.any(Object));
|
||||
expect(options.method).toBe("POST");
|
||||
expect(options.headers).toEqual(expect.objectContaining({
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
}));
|
||||
expect(options.body).toBe(JSON.stringify({ description: "Create me" }));
|
||||
});
|
||||
|
||||
it("listTasks() sends optional query params", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify([]), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
|
||||
await client.listTasks({ column: "in-progress", limit: 10 });
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
`${BASE_URL}/api/tasks?column=in-progress&limit=10`,
|
||||
expect.objectContaining({ method: "GET" })
|
||||
);
|
||||
});
|
||||
|
||||
it("executeTask() posts to execute endpoint", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ acknowledged: true }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
|
||||
const result = await client.executeTask("KB-123");
|
||||
|
||||
expect(result).toEqual({ acknowledged: true });
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
`${BASE_URL}/api/tasks/KB-123/execute`,
|
||||
expect.objectContaining({ method: "POST" })
|
||||
);
|
||||
});
|
||||
|
||||
it("streamEvents() yields parsed events from SSE stream", async () => {
|
||||
const sseBody = [
|
||||
"event: task:created",
|
||||
'data: {"type":"task:created","payload":{"id":"KB-1"},"timestamp":"2026-04-08T00:00:00.000Z"}',
|
||||
"",
|
||||
"event: task:updated",
|
||||
'data: {"type":"task:updated","payload":{"id":"KB-1","column":"in-progress"},"timestamp":"2026-04-08T00:01:00.000Z"}',
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
globalThis.fetch = vi.fn().mockResolvedValue(
|
||||
new Response(sseBody, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
|
||||
|
||||
const events: unknown[] = [];
|
||||
for await (const event of client.streamEvents()) {
|
||||
events.push(event);
|
||||
}
|
||||
|
||||
expect(events).toEqual([
|
||||
{
|
||||
type: "task:created",
|
||||
payload: { id: "KB-1" },
|
||||
timestamp: "2026-04-08T00:00:00.000Z",
|
||||
},
|
||||
{
|
||||
type: "task:updated",
|
||||
payload: { id: "KB-1", column: "in-progress" },
|
||||
timestamp: "2026-04-08T00:01:00.000Z",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("retries on network errors", async () => {
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new TypeError("network down"))
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ status: "ok", version: "1.0.0", uptime: 123 }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
|
||||
|
||||
await expect(client.health()).resolves.toEqual({
|
||||
status: "ok",
|
||||
version: "1.0.0",
|
||||
uptime: 123,
|
||||
});
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("does not retry on 4xx responses", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ error: "unauthorized" }), {
|
||||
status: 401,
|
||||
statusText: "Unauthorized",
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
|
||||
|
||||
await expect(client.health()).rejects.toThrow("401 Unauthorized");
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("retries on 5xx responses", async () => {
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response("server error", { status: 500, statusText: "Internal Server Error" })
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response("server error", { status: 502, statusText: "Bad Gateway" })
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ status: "ok", version: "1.0.0", uptime: 999 }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
|
||||
|
||||
await expect(client.health()).resolves.toEqual({
|
||||
status: "ok",
|
||||
version: "1.0.0",
|
||||
uptime: 999,
|
||||
});
|
||||
expect(fetchMock).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("aborts requests after timeoutMs", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const fetchMock = vi.fn().mockImplementation((_: unknown, init?: RequestInit) => {
|
||||
return new Promise((_resolve, reject) => {
|
||||
const signal = init?.signal;
|
||||
signal?.addEventListener("abort", () => {
|
||||
const abortError = new Error("aborted");
|
||||
abortError.name = "AbortError";
|
||||
reject(abortError);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const client = new RemoteNodeClient({
|
||||
baseUrl: BASE_URL,
|
||||
apiKey: API_KEY,
|
||||
timeoutMs: 5,
|
||||
});
|
||||
|
||||
const request = client.health();
|
||||
const expectation = expect(request).rejects.toThrow("timed out");
|
||||
|
||||
await vi.runAllTimersAsync();
|
||||
await expectation;
|
||||
expect(fetchMock).toHaveBeenCalledTimes(4); // initial + 3 retries
|
||||
});
|
||||
|
||||
it("sends auth header on all request methods", async () => {
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ status: "ok", version: "1.0.0", uptime: 1 }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ inFlightTasks: 0, activeAgents: 0, lastActivityAt: "now" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify([]), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify({ acknowledged: true }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response("event: ping\ndata: {}\n\n", {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
);
|
||||
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
|
||||
|
||||
await client.health();
|
||||
await client.getMetrics();
|
||||
await client.listTasks();
|
||||
await client.executeTask("KB-777");
|
||||
for await (const _event of client.streamEvents()) {
|
||||
// Drain one-response event stream
|
||||
}
|
||||
|
||||
for (const call of fetchMock.mock.calls) {
|
||||
const options = call[1] as RequestInit;
|
||||
expect(options.headers).toEqual(
|
||||
expect.objectContaining({
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
})
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
441
packages/engine/src/runtimes/remote-node-client.ts
Normal file
441
packages/engine/src/runtimes/remote-node-client.ts
Normal file
@@ -0,0 +1,441 @@
|
||||
import type { Task, TaskCreateInput } from "@fusion/core";
|
||||
import type { RuntimeMetrics } from "../project-runtime.js";
|
||||
import { remoteNodeLog } from "../logger.js";
|
||||
|
||||
export interface RemoteNodeEvent {
|
||||
type: string;
|
||||
payload: unknown;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface RemoteNodeClientOptions {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export type RemoteTaskListFilter = Record<string, string | number | boolean | undefined | null>;
|
||||
|
||||
class RemoteNodeRequestError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly retryable: boolean,
|
||||
readonly status?: number
|
||||
) {
|
||||
super(message);
|
||||
this.name = "RemoteNodeRequestError";
|
||||
}
|
||||
}
|
||||
|
||||
const RETRY_BASE_DELAY_MS = 1000;
|
||||
const DEFAULT_MAX_RETRIES = 3;
|
||||
|
||||
export class RemoteNodeClient {
|
||||
private readonly baseUrl: string;
|
||||
private readonly apiKey: string;
|
||||
private readonly timeoutMs: number;
|
||||
|
||||
constructor(options: RemoteNodeClientOptions) {
|
||||
this.baseUrl = options.baseUrl.replace(/\/+$/, "");
|
||||
this.apiKey = options.apiKey;
|
||||
this.timeoutMs = options.timeoutMs ?? 30_000;
|
||||
}
|
||||
|
||||
async health(): Promise<{ status: string; version: string; uptime: number }> {
|
||||
return this.withRetry(() =>
|
||||
this.requestJson<{ status: string; version: string; uptime: number }>("/api/health", {
|
||||
method: "GET",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async getMetrics(): Promise<RuntimeMetrics> {
|
||||
return this.withRetry(() =>
|
||||
this.requestJson<RuntimeMetrics>("/api/metrics", {
|
||||
method: "GET",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async createTask(input: TaskCreateInput): Promise<Task> {
|
||||
return this.withRetry(() =>
|
||||
this.requestJson<Task>("/api/tasks", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(input),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async listTasks(filter?: RemoteTaskListFilter): Promise<Task[]> {
|
||||
const query = new URLSearchParams();
|
||||
if (filter) {
|
||||
for (const [key, value] of Object.entries(filter)) {
|
||||
if (value !== undefined && value !== null) {
|
||||
query.set(key, String(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const path = query.toString().length > 0 ? `/api/tasks?${query.toString()}` : "/api/tasks";
|
||||
return this.withRetry(() =>
|
||||
this.requestJson<Task[]>(path, {
|
||||
method: "GET",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async executeTask(taskId: string): Promise<{ acknowledged: boolean; [key: string]: unknown }> {
|
||||
return this.withRetry(() =>
|
||||
this.requestJson<{ acknowledged: boolean; [key: string]: unknown }>(
|
||||
`/api/tasks/${encodeURIComponent(taskId)}/execute`,
|
||||
{
|
||||
method: "POST",
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
async *streamEvents(options?: { signal?: AbortSignal }): AsyncIterable<RemoteNodeEvent> {
|
||||
const response = await this.withRetry(
|
||||
() => this.openStream("/api/events/stream", options?.signal),
|
||||
DEFAULT_MAX_RETRIES
|
||||
);
|
||||
|
||||
const contentType = response.headers.get("content-type") ?? "";
|
||||
if (!response.body) {
|
||||
throw new Error("Remote node event stream opened without a body");
|
||||
}
|
||||
|
||||
if (contentType.includes("text/event-stream")) {
|
||||
yield* this.parseSseStream(response.body, options?.signal);
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback for long-polling endpoints that return JSON payloads.
|
||||
if (contentType.includes("application/json")) {
|
||||
const payload = (await response.json()) as unknown;
|
||||
if (Array.isArray(payload)) {
|
||||
for (const rawEvent of payload) {
|
||||
yield this.normalizeEvent(rawEvent, "message");
|
||||
}
|
||||
} else {
|
||||
yield this.normalizeEvent(payload, "message");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Generic fallback: treat each line as one JSON event.
|
||||
yield* this.parseJsonLines(response.body, options?.signal);
|
||||
}
|
||||
|
||||
private async requestJson<T>(path: string, init: RequestInit): Promise<T> {
|
||||
const response = await this.fetchWithTimeout(path, {
|
||||
...init,
|
||||
headers: {
|
||||
...this.getAuthHeaders(),
|
||||
Accept: "application/json",
|
||||
...(init.body ? { "Content-Type": "application/json" } : {}),
|
||||
...(init.headers ?? {}),
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
await this.throwHttpError(path, response);
|
||||
}
|
||||
|
||||
try {
|
||||
return (await response.json()) as T;
|
||||
} catch (error) {
|
||||
throw new RemoteNodeRequestError(
|
||||
`Failed to parse JSON response for ${path}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
false
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async openStream(path: string, signal?: AbortSignal): Promise<Response> {
|
||||
const response = await this.fetchWithTimeout(
|
||||
path,
|
||||
{
|
||||
method: "GET",
|
||||
headers: {
|
||||
...this.getAuthHeaders(),
|
||||
Accept: "text/event-stream, application/json",
|
||||
},
|
||||
},
|
||||
signal
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
await this.throwHttpError(path, response);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
private async throwHttpError(path: string, response: Response): Promise<never> {
|
||||
const responseBody = (await response.text()).trim();
|
||||
const snippet = responseBody.length > 0 ? ` — ${responseBody.slice(0, 300)}` : "";
|
||||
const retryable = response.status >= 500;
|
||||
|
||||
throw new RemoteNodeRequestError(
|
||||
`Remote node request failed (${response.status} ${response.statusText}) for ${path}${snippet}`,
|
||||
retryable,
|
||||
response.status
|
||||
);
|
||||
}
|
||||
|
||||
private getAuthHeaders(): Record<string, string> {
|
||||
return {
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
};
|
||||
}
|
||||
|
||||
private async fetchWithTimeout(
|
||||
path: string,
|
||||
init: RequestInit,
|
||||
externalSignal?: AbortSignal
|
||||
): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
let timedOut = false;
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
controller.abort();
|
||||
}, this.timeoutMs);
|
||||
|
||||
const onAbort = () => controller.abort(externalSignal?.reason);
|
||||
if (externalSignal) {
|
||||
if (externalSignal.aborted) {
|
||||
clearTimeout(timeout);
|
||||
throw new RemoteNodeRequestError("Request aborted", false);
|
||||
}
|
||||
externalSignal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
|
||||
try {
|
||||
return await fetch(`${this.baseUrl}${path}`, {
|
||||
...init,
|
||||
signal: controller.signal,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof RemoteNodeRequestError) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (timedOut) {
|
||||
throw new RemoteNodeRequestError(
|
||||
`Remote node request timed out after ${this.timeoutMs}ms (${path})`,
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
if (error instanceof Error && error.name === "AbortError") {
|
||||
throw new RemoteNodeRequestError(`Remote node request aborted (${path})`, false);
|
||||
}
|
||||
|
||||
throw new RemoteNodeRequestError(
|
||||
`Remote node network error (${path}): ${error instanceof Error ? error.message : String(error)}`,
|
||||
true
|
||||
);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
if (externalSignal) {
|
||||
externalSignal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async *parseSseStream(
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
signal?: AbortSignal
|
||||
): AsyncIterable<RemoteNodeEvent> {
|
||||
const reader = stream.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
|
||||
let buffer = "";
|
||||
let eventType = "message";
|
||||
let dataLines: string[] = [];
|
||||
|
||||
const flushEvent = (): RemoteNodeEvent | null => {
|
||||
if (dataLines.length === 0) {
|
||||
eventType = "message";
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = dataLines.join("\n");
|
||||
dataLines = [];
|
||||
|
||||
const normalized = this.normalizeEvent(data, eventType);
|
||||
eventType = "message";
|
||||
return normalized;
|
||||
};
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
if (signal?.aborted) {
|
||||
break;
|
||||
}
|
||||
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split(/\r?\n/);
|
||||
buffer = lines.pop() ?? "";
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.length === 0) {
|
||||
const event = flushEvent();
|
||||
if (event) {
|
||||
yield event;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.startsWith(":")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const separator = line.indexOf(":");
|
||||
const field = separator === -1 ? line : line.slice(0, separator);
|
||||
const valuePart = separator === -1 ? "" : line.slice(separator + 1).trimStart();
|
||||
|
||||
if (field === "event") {
|
||||
eventType = valuePart || "message";
|
||||
} else if (field === "data") {
|
||||
dataLines.push(valuePart);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (buffer.trim().length > 0) {
|
||||
dataLines.push(buffer.trim());
|
||||
}
|
||||
|
||||
const trailingEvent = flushEvent();
|
||||
if (trailingEvent) {
|
||||
yield trailingEvent;
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
private async *parseJsonLines(
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
signal?: AbortSignal
|
||||
): AsyncIterable<RemoteNodeEvent> {
|
||||
const reader = stream.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
if (signal?.aborted) {
|
||||
break;
|
||||
}
|
||||
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split(/\r?\n/);
|
||||
buffer = lines.pop() ?? "";
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
yield this.normalizeEvent(trimmed, "message");
|
||||
}
|
||||
}
|
||||
|
||||
const trailing = buffer.trim();
|
||||
if (trailing.length > 0) {
|
||||
yield this.normalizeEvent(trailing, "message");
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
private normalizeEvent(raw: unknown, fallbackType: string): RemoteNodeEvent {
|
||||
const parsed = this.tryParseJson(raw);
|
||||
|
||||
if (
|
||||
parsed &&
|
||||
typeof parsed === "object" &&
|
||||
"type" in parsed &&
|
||||
"timestamp" in parsed
|
||||
) {
|
||||
return {
|
||||
type: String((parsed as { type: unknown }).type),
|
||||
payload: (parsed as { payload?: unknown }).payload,
|
||||
timestamp: String((parsed as { timestamp: unknown }).timestamp),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
type: fallbackType,
|
||||
payload: parsed,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
private tryParseJson(value: unknown): unknown {
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
private async withRetry<T>(fn: () => Promise<T>, maxRetries = DEFAULT_MAX_RETRIES): Promise<T> {
|
||||
let attempt = 0;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (error) {
|
||||
const isRetryable =
|
||||
error instanceof RemoteNodeRequestError
|
||||
? error.retryable
|
||||
: this.isLikelyNetworkError(error);
|
||||
|
||||
if (!isRetryable || attempt >= maxRetries) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
const delayMs = RETRY_BASE_DELAY_MS * 2 ** attempt;
|
||||
attempt += 1;
|
||||
remoteNodeLog.warn(
|
||||
`Request failed, retrying in ${delayMs}ms (attempt ${attempt}/${maxRetries})`,
|
||||
error
|
||||
);
|
||||
await this.sleep(delayMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private isLikelyNetworkError(error: unknown): boolean {
|
||||
if (!(error instanceof Error)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (error.name === "AbortError") {
|
||||
return true;
|
||||
}
|
||||
|
||||
return error instanceof TypeError;
|
||||
}
|
||||
|
||||
private async sleep(ms: number): Promise<void> {
|
||||
await new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
}
|
||||
266
packages/engine/src/runtimes/remote-node-runtime.test.ts
Normal file
266
packages/engine/src/runtimes/remote-node-runtime.test.ts
Normal file
@@ -0,0 +1,266 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { NodeConfig } from "@fusion/core";
|
||||
import type { RuntimeMetrics } from "../project-runtime.js";
|
||||
import { RemoteNodeRuntime } from "./remote-node-runtime.js";
|
||||
|
||||
const mockClientConstructor = vi.hoisted(() => vi.fn());
|
||||
const mockHealth = vi.hoisted(() => vi.fn());
|
||||
const mockGetMetrics = vi.hoisted(() => vi.fn());
|
||||
const mockStreamEvents = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("./remote-node-client.js", () => ({
|
||||
RemoteNodeClient: vi.fn().mockImplementation((options: unknown) => {
|
||||
mockClientConstructor(options);
|
||||
return {
|
||||
health: mockHealth,
|
||||
getMetrics: mockGetMetrics,
|
||||
streamEvents: mockStreamEvents,
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
const NOW = "2026-04-08T00:00:00.000Z";
|
||||
|
||||
function createNode(overrides?: Partial<NodeConfig>): NodeConfig {
|
||||
return {
|
||||
id: "node_remote_1",
|
||||
name: "Remote Node",
|
||||
type: "remote",
|
||||
url: "https://remote.example.com",
|
||||
apiKey: "token-123",
|
||||
status: "online",
|
||||
maxConcurrent: 4,
|
||||
createdAt: NOW,
|
||||
updatedAt: NOW,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function* idleStream(signal?: AbortSignal): AsyncIterable<unknown> {
|
||||
while (!signal?.aborted) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
}
|
||||
|
||||
async function* eventStream(events: unknown[], signal?: AbortSignal): AsyncIterable<unknown> {
|
||||
for (const event of events) {
|
||||
yield event;
|
||||
}
|
||||
|
||||
while (!signal?.aborted) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
}
|
||||
|
||||
describe("RemoteNodeRuntime", () => {
|
||||
beforeEach(() => {
|
||||
mockClientConstructor.mockReset();
|
||||
mockHealth.mockReset();
|
||||
mockGetMetrics.mockReset();
|
||||
mockStreamEvents.mockReset();
|
||||
|
||||
mockHealth.mockResolvedValue({ status: "ok", version: "1.0.0", uptime: 100 });
|
||||
mockGetMetrics.mockResolvedValue({
|
||||
inFlightTasks: 1,
|
||||
activeAgents: 2,
|
||||
lastActivityAt: NOW,
|
||||
} satisfies RuntimeMetrics);
|
||||
mockStreamEvents.mockImplementation(({ signal }: { signal?: AbortSignal } = {}) =>
|
||||
idleStream(signal)
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("start() transitions stopped -> starting -> active and starts stream", async () => {
|
||||
const runtime = new RemoteNodeRuntime({
|
||||
nodeConfig: createNode(),
|
||||
projectId: "proj_1",
|
||||
projectName: "Project 1",
|
||||
});
|
||||
|
||||
const healthEvents: string[] = [];
|
||||
runtime.on("health-changed", ({ status }) => {
|
||||
healthEvents.push(status);
|
||||
});
|
||||
|
||||
await runtime.start();
|
||||
|
||||
expect(runtime.getStatus()).toBe("active");
|
||||
expect(healthEvents).toEqual(["starting", "active"]);
|
||||
expect(mockHealth).toHaveBeenCalled();
|
||||
expect(mockStreamEvents).toHaveBeenCalled();
|
||||
expect(mockClientConstructor).toHaveBeenCalledWith({
|
||||
baseUrl: "https://remote.example.com",
|
||||
apiKey: "token-123",
|
||||
});
|
||||
|
||||
await runtime.stop();
|
||||
});
|
||||
|
||||
it("stop() transitions to stopped and is idempotent", async () => {
|
||||
const runtime = new RemoteNodeRuntime({
|
||||
nodeConfig: createNode(),
|
||||
projectId: "proj_2",
|
||||
projectName: "Project 2",
|
||||
});
|
||||
|
||||
await runtime.start();
|
||||
await runtime.stop();
|
||||
|
||||
expect(runtime.getStatus()).toBe("stopped");
|
||||
|
||||
await expect(runtime.stop()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("getTaskStore() throws descriptive error", () => {
|
||||
const runtime = new RemoteNodeRuntime({
|
||||
nodeConfig: createNode(),
|
||||
projectId: "proj_3",
|
||||
projectName: "Project 3",
|
||||
});
|
||||
|
||||
expect(() => runtime.getTaskStore()).toThrow(
|
||||
"TaskStore not accessible for remote node runtime"
|
||||
);
|
||||
});
|
||||
|
||||
it("getScheduler() throws descriptive error", () => {
|
||||
const runtime = new RemoteNodeRuntime({
|
||||
nodeConfig: createNode(),
|
||||
projectId: "proj_4",
|
||||
projectName: "Project 4",
|
||||
});
|
||||
|
||||
expect(() => runtime.getScheduler()).toThrow("Scheduler not accessible for remote node runtime");
|
||||
});
|
||||
|
||||
it("getMetrics() returns fetched metrics on success and fallback on failure", async () => {
|
||||
const runtime = new RemoteNodeRuntime({
|
||||
nodeConfig: createNode(),
|
||||
projectId: "proj_5",
|
||||
projectName: "Project 5",
|
||||
});
|
||||
|
||||
await runtime.start();
|
||||
|
||||
expect(runtime.getMetrics()).toEqual({
|
||||
inFlightTasks: 1,
|
||||
activeAgents: 2,
|
||||
lastActivityAt: NOW,
|
||||
});
|
||||
|
||||
mockGetMetrics.mockRejectedValueOnce(new Error("metrics unavailable"));
|
||||
|
||||
runtime.getMetrics();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(runtime.getMetrics()).toEqual({
|
||||
inFlightTasks: 0,
|
||||
activeAgents: 0,
|
||||
lastActivityAt: NOW,
|
||||
});
|
||||
|
||||
await runtime.stop();
|
||||
});
|
||||
|
||||
it("forwards remote task and error events", async () => {
|
||||
const createdHandler = vi.fn();
|
||||
const movedHandler = vi.fn();
|
||||
const updatedHandler = vi.fn();
|
||||
const errorHandler = vi.fn();
|
||||
|
||||
mockStreamEvents.mockImplementation(({ signal }: { signal?: AbortSignal } = {}) =>
|
||||
eventStream(
|
||||
[
|
||||
{
|
||||
type: "task:created",
|
||||
payload: { id: "KB-1" },
|
||||
timestamp: NOW,
|
||||
},
|
||||
{
|
||||
type: "task:moved",
|
||||
payload: { task: { id: "KB-1" }, from: "todo", to: "in-progress" },
|
||||
timestamp: NOW,
|
||||
},
|
||||
{
|
||||
type: "task:updated",
|
||||
payload: { id: "KB-1", column: "done" },
|
||||
timestamp: NOW,
|
||||
},
|
||||
{
|
||||
type: "error",
|
||||
payload: { message: "boom" },
|
||||
timestamp: NOW,
|
||||
},
|
||||
],
|
||||
signal
|
||||
)
|
||||
);
|
||||
|
||||
const runtime = new RemoteNodeRuntime({
|
||||
nodeConfig: createNode(),
|
||||
projectId: "proj_6",
|
||||
projectName: "Project 6",
|
||||
});
|
||||
|
||||
runtime.on("task:created", createdHandler);
|
||||
runtime.on("task:moved", movedHandler);
|
||||
runtime.on("task:updated", updatedHandler);
|
||||
runtime.on("error", errorHandler);
|
||||
|
||||
await runtime.start();
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(createdHandler).toHaveBeenCalledWith({ id: "KB-1" });
|
||||
expect(movedHandler).toHaveBeenCalledWith({
|
||||
task: { id: "KB-1" },
|
||||
from: "todo",
|
||||
to: "in-progress",
|
||||
});
|
||||
expect(updatedHandler).toHaveBeenCalledWith({ id: "KB-1", column: "done" });
|
||||
expect(errorHandler).toHaveBeenCalledWith(expect.any(Error));
|
||||
});
|
||||
|
||||
await runtime.stop();
|
||||
});
|
||||
|
||||
it("reconnects when stream ends unexpectedly and transitions to errored after max attempts", async () => {
|
||||
mockStreamEvents.mockImplementation(async function* () {
|
||||
// Immediate end to force reconnect loop.
|
||||
});
|
||||
|
||||
const runtime = new RemoteNodeRuntime({
|
||||
nodeConfig: createNode(),
|
||||
projectId: "proj_7",
|
||||
projectName: "Project 7",
|
||||
});
|
||||
|
||||
(runtime as unknown as { reconnectBaseDelayMs: number }).reconnectBaseDelayMs = 1;
|
||||
(runtime as unknown as { maxReconnectDelayMs: number }).maxReconnectDelayMs = 1;
|
||||
(runtime as unknown as { maxReconnectAttempts: number }).maxReconnectAttempts = 3;
|
||||
|
||||
await runtime.start();
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(runtime.getStatus()).toBe("errored");
|
||||
});
|
||||
|
||||
expect(mockStreamEvents.mock.calls.length).toBeGreaterThanOrEqual(3);
|
||||
|
||||
await runtime.stop();
|
||||
});
|
||||
|
||||
it("validates remote node config on start", async () => {
|
||||
const runtime = new RemoteNodeRuntime({
|
||||
nodeConfig: createNode({ type: "local", url: undefined, apiKey: undefined }),
|
||||
projectId: "proj_8",
|
||||
projectName: "Project 8",
|
||||
});
|
||||
|
||||
await expect(runtime.start()).rejects.toThrow("requires a remote node configuration");
|
||||
});
|
||||
});
|
||||
343
packages/engine/src/runtimes/remote-node-runtime.ts
Normal file
343
packages/engine/src/runtimes/remote-node-runtime.ts
Normal file
@@ -0,0 +1,343 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import type { NodeConfig, Task, TaskStore } from "@fusion/core";
|
||||
import type { Scheduler } from "../scheduler.js";
|
||||
import type {
|
||||
ProjectRuntime,
|
||||
ProjectRuntimeEvents,
|
||||
RuntimeMetrics,
|
||||
RuntimeStatus,
|
||||
} from "../project-runtime.js";
|
||||
import { remoteNodeLog } from "../logger.js";
|
||||
import { RemoteNodeClient, type RemoteNodeEvent } from "./remote-node-client.js";
|
||||
|
||||
export interface RemoteNodeRuntimeConfig {
|
||||
nodeConfig: NodeConfig;
|
||||
projectId: string;
|
||||
projectName: string;
|
||||
}
|
||||
|
||||
export class RemoteNodeRuntime
|
||||
extends EventEmitter<ProjectRuntimeEvents>
|
||||
implements ProjectRuntime
|
||||
{
|
||||
private status: RuntimeStatus = "stopped";
|
||||
private client: RemoteNodeClient;
|
||||
private healthInterval: ReturnType<typeof setInterval> | null = null;
|
||||
private streamLoopAbortController: AbortController | null = null;
|
||||
private streamLoopPromise: Promise<void> | null = null;
|
||||
private lastSuccessfulMetricsAt: string;
|
||||
private cachedMetrics: RuntimeMetrics;
|
||||
|
||||
// Kept as mutable fields for testability.
|
||||
private reconnectBaseDelayMs = 5_000;
|
||||
private maxReconnectDelayMs = 60_000;
|
||||
private maxReconnectAttempts = 10;
|
||||
|
||||
constructor(private config: RemoteNodeRuntimeConfig) {
|
||||
super();
|
||||
this.setMaxListeners(100);
|
||||
|
||||
this.cachedMetrics = {
|
||||
inFlightTasks: 0,
|
||||
activeAgents: 0,
|
||||
lastActivityAt: new Date().toISOString(),
|
||||
};
|
||||
this.lastSuccessfulMetricsAt = this.cachedMetrics.lastActivityAt;
|
||||
|
||||
this.client = new RemoteNodeClient({
|
||||
baseUrl: config.nodeConfig.url ?? "",
|
||||
apiKey: config.nodeConfig.apiKey ?? "",
|
||||
});
|
||||
|
||||
remoteNodeLog.log(
|
||||
`Created RemoteNodeRuntime for project ${config.projectId} on node ${config.nodeConfig.name}`
|
||||
);
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
if (this.status !== "stopped") {
|
||||
throw new Error(`Cannot start runtime: current status is ${this.status}`);
|
||||
}
|
||||
|
||||
this.validateRemoteNodeConfig();
|
||||
|
||||
this.setStatus("starting");
|
||||
try {
|
||||
await this.client.health();
|
||||
await this.refreshMetrics();
|
||||
|
||||
this.setStatus("active");
|
||||
this.startHealthChecks();
|
||||
this.startEventStreamLoop();
|
||||
|
||||
remoteNodeLog.log(
|
||||
`RemoteNodeRuntime started for ${this.config.projectId} (${this.config.projectName})`
|
||||
);
|
||||
} catch (error) {
|
||||
const err = this.toError(error);
|
||||
this.setStatus("errored");
|
||||
this.emitRuntimeError(err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
if (this.status === "stopped" || this.status === "stopping") {
|
||||
return;
|
||||
}
|
||||
|
||||
this.setStatus("stopping");
|
||||
|
||||
if (this.healthInterval) {
|
||||
clearInterval(this.healthInterval);
|
||||
this.healthInterval = null;
|
||||
}
|
||||
|
||||
if (this.streamLoopAbortController) {
|
||||
this.streamLoopAbortController.abort();
|
||||
}
|
||||
|
||||
if (this.streamLoopPromise) {
|
||||
try {
|
||||
await this.streamLoopPromise;
|
||||
} catch {
|
||||
// Best-effort shutdown. Errors are already emitted via runtime events.
|
||||
}
|
||||
}
|
||||
|
||||
this.streamLoopAbortController = null;
|
||||
this.streamLoopPromise = null;
|
||||
|
||||
this.setStatus("stopped");
|
||||
remoteNodeLog.log(`RemoteNodeRuntime stopped for ${this.config.projectId}`);
|
||||
}
|
||||
|
||||
getStatus(): RuntimeStatus {
|
||||
return this.status;
|
||||
}
|
||||
|
||||
getTaskStore(): TaskStore {
|
||||
throw new Error(
|
||||
"TaskStore not accessible for remote node runtime. Use the remote Fusion API directly."
|
||||
);
|
||||
}
|
||||
|
||||
getScheduler(): Scheduler {
|
||||
throw new Error("Scheduler not accessible for remote node runtime.");
|
||||
}
|
||||
|
||||
getMetrics(): RuntimeMetrics {
|
||||
void this.refreshMetrics();
|
||||
return { ...this.cachedMetrics };
|
||||
}
|
||||
|
||||
private validateRemoteNodeConfig(): void {
|
||||
if (this.config.nodeConfig.type !== "remote") {
|
||||
throw new Error(
|
||||
`RemoteNodeRuntime requires a remote node configuration (received: ${this.config.nodeConfig.type})`
|
||||
);
|
||||
}
|
||||
|
||||
if (!this.config.nodeConfig.url) {
|
||||
throw new Error("Remote node runtime requires nodeConfig.url for remote nodes.");
|
||||
}
|
||||
|
||||
if (!this.config.nodeConfig.apiKey) {
|
||||
throw new Error("Remote node runtime requires nodeConfig.apiKey for authentication.");
|
||||
}
|
||||
}
|
||||
|
||||
private startHealthChecks(): void {
|
||||
if (this.healthInterval) {
|
||||
clearInterval(this.healthInterval);
|
||||
}
|
||||
|
||||
this.healthInterval = setInterval(() => {
|
||||
void this.client.health().catch((error) => {
|
||||
this.emitRuntimeError(this.toError(error));
|
||||
});
|
||||
}, 30_000);
|
||||
}
|
||||
|
||||
private startEventStreamLoop(): void {
|
||||
if (this.streamLoopPromise) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.streamLoopAbortController = new AbortController();
|
||||
this.streamLoopPromise = this.runEventStreamLoop(this.streamLoopAbortController.signal)
|
||||
.catch((error) => {
|
||||
this.emitRuntimeError(this.toError(error));
|
||||
})
|
||||
.finally(() => {
|
||||
this.streamLoopPromise = null;
|
||||
});
|
||||
}
|
||||
|
||||
private async runEventStreamLoop(signal: AbortSignal): Promise<void> {
|
||||
let reconnectAttempts = 0;
|
||||
|
||||
while (!signal.aborted && !this.isShuttingDown()) {
|
||||
let sawAnyEvent = false;
|
||||
|
||||
try {
|
||||
for await (const event of this.client.streamEvents({ signal })) {
|
||||
if (signal.aborted || this.isShuttingDown()) {
|
||||
return;
|
||||
}
|
||||
|
||||
sawAnyEvent = true;
|
||||
this.forwardRemoteEvent(event);
|
||||
}
|
||||
|
||||
if (signal.aborted || this.isShuttingDown()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (sawAnyEvent) {
|
||||
reconnectAttempts = 0;
|
||||
}
|
||||
|
||||
throw new Error("Remote event stream ended unexpectedly");
|
||||
} catch (error) {
|
||||
if (signal.aborted || this.isShuttingDown()) {
|
||||
return;
|
||||
}
|
||||
|
||||
reconnectAttempts += 1;
|
||||
this.emitRuntimeError(this.toError(error));
|
||||
|
||||
if (reconnectAttempts >= this.maxReconnectAttempts) {
|
||||
this.setStatus("errored");
|
||||
return;
|
||||
}
|
||||
|
||||
const delayMs = Math.min(
|
||||
this.reconnectBaseDelayMs * 2 ** (reconnectAttempts - 1),
|
||||
this.maxReconnectDelayMs
|
||||
);
|
||||
|
||||
remoteNodeLog.warn(
|
||||
`Remote event stream disconnected for ${this.config.projectId}; reconnecting in ${delayMs}ms ` +
|
||||
`(attempt ${reconnectAttempts}/${this.maxReconnectAttempts})`
|
||||
);
|
||||
|
||||
await this.sleep(delayMs, signal);
|
||||
if (signal.aborted) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.client.health();
|
||||
} catch (healthError) {
|
||||
this.emitRuntimeError(this.toError(healthError));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private forwardRemoteEvent(event: RemoteNodeEvent): void {
|
||||
switch (event.type) {
|
||||
case "task:created":
|
||||
this.emit("task:created", event.payload as Task);
|
||||
break;
|
||||
case "task:moved": {
|
||||
const payload = event.payload as { task: Task; from: string; to: string };
|
||||
this.emit("task:moved", {
|
||||
task: payload.task,
|
||||
from: payload.from,
|
||||
to: payload.to,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "task:updated":
|
||||
this.emit("task:updated", event.payload as Task);
|
||||
break;
|
||||
case "error": {
|
||||
const payload = event.payload;
|
||||
if (payload instanceof Error) {
|
||||
this.emitRuntimeError(payload);
|
||||
} else if (typeof payload === "object" && payload && "message" in payload) {
|
||||
const message = String((payload as { message: unknown }).message);
|
||||
this.emitRuntimeError(new Error(message));
|
||||
} else {
|
||||
this.emitRuntimeError(new Error(String(payload)));
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
remoteNodeLog.warn(
|
||||
`Ignoring unsupported remote event type "${event.type}" for ${this.config.projectId}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async refreshMetrics(): Promise<RuntimeMetrics> {
|
||||
try {
|
||||
const metrics = await this.client.getMetrics();
|
||||
this.cachedMetrics = { ...metrics };
|
||||
this.lastSuccessfulMetricsAt = metrics.lastActivityAt;
|
||||
return metrics;
|
||||
} catch {
|
||||
const fallback: RuntimeMetrics = {
|
||||
inFlightTasks: 0,
|
||||
activeAgents: 0,
|
||||
lastActivityAt: this.lastSuccessfulMetricsAt,
|
||||
};
|
||||
this.cachedMetrics = fallback;
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
private setStatus(newStatus: RuntimeStatus): void {
|
||||
const previous = this.status;
|
||||
this.status = newStatus;
|
||||
|
||||
if (previous !== newStatus) {
|
||||
this.emit("health-changed", {
|
||||
status: newStatus,
|
||||
previous,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private emitRuntimeError(error: Error): void {
|
||||
if (this.listenerCount("error") > 0) {
|
||||
this.emit("error", error);
|
||||
return;
|
||||
}
|
||||
|
||||
remoteNodeLog.error(
|
||||
`Unhandled remote runtime error for ${this.config.projectId}: ${error.message}`
|
||||
);
|
||||
}
|
||||
|
||||
private isShuttingDown(): boolean {
|
||||
return this.status === "stopping" || this.status === "stopped";
|
||||
}
|
||||
|
||||
private toError(error: unknown): Error {
|
||||
return error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
|
||||
private async sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
await new Promise<void>((resolve) => {
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve();
|
||||
}, ms);
|
||||
|
||||
const onAbort = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
};
|
||||
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user