feat(KB-616): add ProjectRuntime abstraction for task execution isolation
- Define ProjectRuntime interface with unified task execution contract\n- Implement IPC protocol for host-worker communication (messages, streaming, heartbeats)\n- Add InProcessRuntime for synchronous in-process task execution\n- Add ChildProcessRuntime with isolated worker processes for sandboxed execution\n- Implement ProjectManager to coordinate runtime selection and task lifecycle\n- Update engine exports to expose runtime APIs\n- Add comprehensive tests for all runtime implementations and IPC protocol
This commit is contained in:
gsxdsm
499
packages/engine/src/runtimes/child-process-runtime.ts
Normal file
499
packages/engine/src/runtimes/child-process-runtime.ts
Normal file
@@ -0,0 +1,499 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { fork, type ChildProcess } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { dirname, join } from "node:path";
|
||||
import type {
|
||||
Task,
|
||||
TaskStore,
|
||||
CentralCore,
|
||||
} from "@fusion/core";
|
||||
import type { Scheduler } from "../scheduler.js";
|
||||
import type {
|
||||
ProjectRuntime,
|
||||
ProjectRuntimeConfig,
|
||||
RuntimeStatus,
|
||||
RuntimeMetrics,
|
||||
ProjectRuntimeEvents,
|
||||
} from "../project-runtime.js";
|
||||
import { IpcHost } from "../ipc/ipc-host.js";
|
||||
import {
|
||||
START_RUNTIME,
|
||||
STOP_RUNTIME,
|
||||
GET_STATUS,
|
||||
GET_METRICS,
|
||||
TASK_CREATED,
|
||||
TASK_MOVED,
|
||||
TASK_UPDATED,
|
||||
ERROR_EVENT,
|
||||
HEALTH_CHANGED,
|
||||
type TaskCreatedPayload,
|
||||
type TaskMovedPayload,
|
||||
type TaskUpdatedPayload,
|
||||
type ErrorEventPayload,
|
||||
type HealthChangedPayload,
|
||||
} from "../ipc/ipc-protocol.js";
|
||||
import { runtimeLog } from "../logger.js";
|
||||
|
||||
/**
|
||||
* Health monitor for tracking child process health.
|
||||
*/
|
||||
class HealthMonitor {
|
||||
private running = false;
|
||||
private missedHeartbeats = 0;
|
||||
private interval: ReturnType<typeof setInterval> | null = null;
|
||||
private restartAttempts = 0;
|
||||
private restartDelays = [1000, 5000, 15000]; // Exponential backoff: 1s, 5s, 15s
|
||||
|
||||
constructor(
|
||||
private onHealthCheck: () => Promise<boolean>,
|
||||
private onUnhealthy: () => void,
|
||||
private options: {
|
||||
intervalMs?: number;
|
||||
maxMissedHeartbeats?: number;
|
||||
maxRestartAttempts?: number;
|
||||
} = {}
|
||||
) {}
|
||||
|
||||
start(): void {
|
||||
if (this.running) return;
|
||||
this.running = true;
|
||||
|
||||
const intervalMs = this.options.intervalMs ?? 5000;
|
||||
const maxMissed = this.options.maxMissedHeartbeats ?? 3;
|
||||
|
||||
this.interval = setInterval(async () => {
|
||||
const healthy = await this.onHealthCheck();
|
||||
|
||||
if (healthy) {
|
||||
if (this.missedHeartbeats > 0) {
|
||||
runtimeLog.log(`Health recovered after ${this.missedHeartbeats} missed heartbeats`);
|
||||
}
|
||||
this.missedHeartbeats = 0;
|
||||
this.restartAttempts = 0; // Reset restart attempts on success
|
||||
} else {
|
||||
this.missedHeartbeats++;
|
||||
runtimeLog.warn(`Missed heartbeat ${this.missedHeartbeats}/${maxMissed}`);
|
||||
|
||||
if (this.missedHeartbeats >= maxMissed) {
|
||||
runtimeLog.error(`Health check failed after ${maxMissed} attempts`);
|
||||
this.onUnhealthy();
|
||||
}
|
||||
}
|
||||
}, intervalMs);
|
||||
|
||||
runtimeLog.log(`Health monitor started (interval: ${intervalMs}ms)`);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.running = false;
|
||||
if (this.interval) {
|
||||
clearInterval(this.interval);
|
||||
this.interval = null;
|
||||
}
|
||||
this.missedHeartbeats = 0;
|
||||
runtimeLog.log("Health monitor stopped");
|
||||
}
|
||||
|
||||
getRestartDelay(): number {
|
||||
const delay = this.restartDelays[this.restartAttempts] ?? this.restartDelays[this.restartDelays.length - 1];
|
||||
return delay;
|
||||
}
|
||||
|
||||
incrementRestartAttempts(): void {
|
||||
this.restartAttempts++;
|
||||
}
|
||||
|
||||
getRestartAttempts(): number {
|
||||
return this.restartAttempts;
|
||||
}
|
||||
|
||||
getMissedHeartbeats(): number {
|
||||
return this.missedHeartbeats;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* ChildProcessRuntime runs a project in an isolated child process.
|
||||
*
|
||||
* This provides stronger isolation between projects at the cost of
|
||||
* IPC overhead. The child process runs an InProcessRuntime internally
|
||||
* and communicates with the host via IPC messages.
|
||||
*
|
||||
* Features:
|
||||
* - Process isolation (separate memory space)
|
||||
* - Automatic restart on crash with exponential backoff
|
||||
* - Health monitoring via heartbeat protocol
|
||||
* - Graceful shutdown with configurable timeout
|
||||
* - Event forwarding from child process to host listeners
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const config: ProjectRuntimeConfig = {
|
||||
* projectId: "proj_abc123",
|
||||
* workingDirectory: "/path/to/project",
|
||||
* isolationMode: "child-process",
|
||||
* maxConcurrent: 2,
|
||||
* maxWorktrees: 4,
|
||||
* };
|
||||
*
|
||||
* const runtime = new ChildProcessRuntime(config, centralCore);
|
||||
* await runtime.start();
|
||||
*
|
||||
* // Access metrics via IPC
|
||||
* const metrics = runtime.getMetrics();
|
||||
*
|
||||
* await runtime.stop();
|
||||
* ```
|
||||
*/
|
||||
export class ChildProcessRuntime
|
||||
extends EventEmitter<ProjectRuntimeEvents>
|
||||
implements ProjectRuntime
|
||||
{
|
||||
private status: RuntimeStatus = "stopped";
|
||||
private child: ChildProcess | null = null;
|
||||
private ipcHost: IpcHost | null = null;
|
||||
private healthMonitor: HealthMonitor;
|
||||
private lastMetrics: RuntimeMetrics = {
|
||||
inFlightTasks: 0,
|
||||
activeAgents: 0,
|
||||
lastActivityAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
/**
|
||||
* @param config - Runtime configuration
|
||||
* @param centralCore - CentralCore reference for global coordination
|
||||
*/
|
||||
constructor(
|
||||
private config: ProjectRuntimeConfig,
|
||||
private centralCore: CentralCore
|
||||
) {
|
||||
super();
|
||||
this.setMaxListeners(100);
|
||||
|
||||
// Initialize health monitor
|
||||
this.healthMonitor = new HealthMonitor(
|
||||
async () => this.checkHealth(),
|
||||
() => this.handleUnhealthy(),
|
||||
{ intervalMs: 5000, maxMissedHeartbeats: 3, maxRestartAttempts: 3 }
|
||||
);
|
||||
|
||||
runtimeLog.log(`Created ChildProcessRuntime for project ${config.projectId}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the runtime by spawning a child process.
|
||||
*
|
||||
* Startup sequence:
|
||||
* 1. Set status to "starting"
|
||||
* 2. Fork child process pointing to worker entry point
|
||||
* 3. Set up IPC host with the child process
|
||||
* 4. Send START_RUNTIME command with serialized config
|
||||
* 5. Wait for OK response or timeout (10s)
|
||||
* 6. Start health monitoring heartbeat
|
||||
* 7. Set status to "active"
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (this.status !== "stopped") {
|
||||
throw new Error(`Cannot start runtime: current status is ${this.status}`);
|
||||
}
|
||||
|
||||
this.setStatus("starting");
|
||||
runtimeLog.log(`Starting ChildProcessRuntime for project ${this.config.projectId}`);
|
||||
|
||||
try {
|
||||
await this.spawnChild();
|
||||
this.setStatus("active");
|
||||
runtimeLog.log(`ChildProcessRuntime started for project ${this.config.projectId}`);
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
this.setStatus("errored");
|
||||
runtimeLog.error(`Failed to start ChildProcessRuntime:`, err.message);
|
||||
this.emit("error", err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawn the child process and set up IPC.
|
||||
*/
|
||||
private async spawnChild(): Promise<void> {
|
||||
// Determine worker entry point
|
||||
const workerPath = this.getWorkerPath();
|
||||
|
||||
runtimeLog.log(`Forking child process: ${workerPath}`);
|
||||
|
||||
// Fork child process
|
||||
this.child = fork(workerPath, [], {
|
||||
silent: true, // Pipe stdout/stderr
|
||||
execArgv: [], // Don't inherit exec arguments
|
||||
});
|
||||
|
||||
// Set up IPC host
|
||||
this.ipcHost = new IpcHost(this.child, { commandTimeoutMs: 10000 });
|
||||
|
||||
// Set up event forwarding
|
||||
this.setupEventForwarding();
|
||||
|
||||
// Send START_RUNTIME command
|
||||
runtimeLog.log("Sending START_RUNTIME command to child");
|
||||
await this.ipcHost.sendCommand(START_RUNTIME, { config: this.config });
|
||||
|
||||
// Start health monitoring
|
||||
this.healthMonitor.start();
|
||||
|
||||
// Handle child process exit
|
||||
this.child.on("exit", (code, signal) => {
|
||||
runtimeLog.warn(`Child process exited (code: ${code}, signal: ${signal})`);
|
||||
this.handleChildExit(code, signal);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path to the worker entry point.
|
||||
*/
|
||||
private getWorkerPath(): string {
|
||||
// In production, use the compiled .js file
|
||||
// In development/tests, use .ts with tsx
|
||||
const isCompiled = !import.meta.url.endsWith(".ts");
|
||||
|
||||
const currentDir = dirname(fileURLToPath(import.meta.url));
|
||||
const workerFile = isCompiled ? "child-process-worker.js" : "child-process-worker.ts";
|
||||
|
||||
return join(currentDir, workerFile);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up event forwarding from IPC host to runtime listeners.
|
||||
*/
|
||||
private setupEventForwarding(): void {
|
||||
if (!this.ipcHost) return;
|
||||
|
||||
// Forward task events
|
||||
this.ipcHost.on(TASK_CREATED, (payload: TaskCreatedPayload) => {
|
||||
this.emit("task:created", payload.task);
|
||||
});
|
||||
|
||||
this.ipcHost.on(TASK_MOVED, (payload: TaskMovedPayload) => {
|
||||
this.emit("task:moved", { task: payload.task, from: payload.from, to: payload.to });
|
||||
});
|
||||
|
||||
this.ipcHost.on(TASK_UPDATED, (payload: TaskUpdatedPayload) => {
|
||||
this.emit("task:updated", payload.task);
|
||||
});
|
||||
|
||||
// Forward error events
|
||||
this.ipcHost.on(ERROR_EVENT, (payload: ErrorEventPayload) => {
|
||||
const error = new Error(payload.message);
|
||||
if (payload.code) {
|
||||
(error as Error & { code: string }).code = payload.code;
|
||||
}
|
||||
this.emit("error", error);
|
||||
});
|
||||
|
||||
// Forward health change events
|
||||
this.ipcHost.on(HEALTH_CHANGED, (payload: HealthChangedPayload) => {
|
||||
this.status = payload.status as RuntimeStatus;
|
||||
this.emit("health-changed", { status: payload.status, previous: payload.previous });
|
||||
});
|
||||
|
||||
// Handle disconnect
|
||||
this.ipcHost.on("disconnect", () => {
|
||||
runtimeLog.warn("IPC host disconnected");
|
||||
this.handleDisconnection();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the runtime with graceful shutdown.
|
||||
*
|
||||
* Shutdown sequence:
|
||||
* 1. Set status to "stopping"
|
||||
* 2. Stop health monitoring
|
||||
* 3. Send STOP_RUNTIME command with 30s timeout
|
||||
* 4. Kill child process if graceful shutdown fails
|
||||
* 5. Set status to "stopped"
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
if (this.status === "stopped" || this.status === "stopping") {
|
||||
return;
|
||||
}
|
||||
|
||||
this.setStatus("stopping");
|
||||
runtimeLog.log(`Stopping ChildProcessRuntime for project ${this.config.projectId}`);
|
||||
|
||||
// Stop health monitoring
|
||||
this.healthMonitor.stop();
|
||||
|
||||
try {
|
||||
// Send graceful shutdown command
|
||||
if (this.ipcHost?.isConnected()) {
|
||||
runtimeLog.log("Sending STOP_RUNTIME command to child");
|
||||
await this.ipcHost.sendCommand(STOP_RUNTIME, { timeoutMs: 30000 }, 35000);
|
||||
}
|
||||
} catch (error) {
|
||||
runtimeLog.warn(`Graceful shutdown failed: ${error}`);
|
||||
}
|
||||
|
||||
// Kill child process if still running
|
||||
this.killChild();
|
||||
|
||||
this.setStatus("stopped");
|
||||
runtimeLog.log(`ChildProcessRuntime stopped for project ${this.config.projectId}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Kill the child process forcefully.
|
||||
*/
|
||||
private killChild(): void {
|
||||
if (this.child && !this.child.killed) {
|
||||
runtimeLog.log("Killing child process");
|
||||
this.child.kill("SIGTERM");
|
||||
|
||||
// Force kill after 5 seconds if still running
|
||||
setTimeout(() => {
|
||||
if (this.child && !this.child.killed) {
|
||||
runtimeLog.warn("Force killing child process");
|
||||
this.child.kill("SIGKILL");
|
||||
}
|
||||
}, 5000);
|
||||
}
|
||||
|
||||
this.child = null;
|
||||
this.ipcHost = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current runtime status.
|
||||
*/
|
||||
getStatus(): RuntimeStatus {
|
||||
return this.status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the project's TaskStore instance.
|
||||
* @throws Error - Not accessible in child mode (use IPC instead)
|
||||
*/
|
||||
getTaskStore(): TaskStore {
|
||||
throw new Error(
|
||||
"TaskStore is not accessible in ChildProcessRuntime. " +
|
||||
"Use IPC methods to access task data."
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the project's Scheduler instance.
|
||||
* @throws Error - Not accessible in child mode
|
||||
*/
|
||||
getScheduler(): Scheduler {
|
||||
throw new Error(
|
||||
"Scheduler is not accessible in ChildProcessRuntime. " +
|
||||
"Use IPC methods to interact with the scheduler."
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current runtime metrics (via IPC query).
|
||||
*/
|
||||
getMetrics(): RuntimeMetrics {
|
||||
// Query metrics via IPC if connected
|
||||
if (this.ipcHost?.isConnected()) {
|
||||
// Fire-and-forget metrics request - returns cached value immediately
|
||||
this.ipcHost
|
||||
.sendCommand(GET_METRICS, {})
|
||||
.then((metrics: unknown) => {
|
||||
this.lastMetrics = metrics as RuntimeMetrics;
|
||||
})
|
||||
.catch(() => {
|
||||
// Ignore errors, use cached value
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...this.lastMetrics,
|
||||
lastActivityAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check health by pinging the child process.
|
||||
*/
|
||||
private async checkHealth(): Promise<boolean> {
|
||||
if (!this.ipcHost?.isConnected()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.ipcHost.ping(5000);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle unhealthy child process (restart or error).
|
||||
*/
|
||||
private handleUnhealthy(): void {
|
||||
const maxRestarts = 3;
|
||||
|
||||
if (this.healthMonitor.getRestartAttempts() >= maxRestarts) {
|
||||
runtimeLog.error(`Max restart attempts (${maxRestarts}) reached, transitioning to errored`);
|
||||
this.setStatus("errored");
|
||||
this.emit("error", new Error("Child process failed after max restart attempts"));
|
||||
return;
|
||||
}
|
||||
|
||||
const delay = this.healthMonitor.getRestartDelay();
|
||||
this.healthMonitor.incrementRestartAttempts();
|
||||
|
||||
runtimeLog.log(`Attempting restart ${this.healthMonitor.getRestartAttempts()}/${maxRestarts} after ${delay}ms`);
|
||||
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
this.killChild();
|
||||
await this.spawnChild();
|
||||
runtimeLog.log("Child process restarted successfully");
|
||||
} catch (error) {
|
||||
runtimeLog.error("Failed to restart child process:", error);
|
||||
this.setStatus("errored");
|
||||
this.emit("error", error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
}, delay);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle child process exit.
|
||||
*/
|
||||
private handleChildExit(code: number | null, signal: string | null): void {
|
||||
// Don't restart if we're intentionally stopping
|
||||
if (this.status === "stopping" || this.status === "stopped") {
|
||||
return;
|
||||
}
|
||||
|
||||
// Unexpected exit - trigger restart
|
||||
runtimeLog.warn(`Unexpected child exit (code: ${code}, signal: ${signal})`);
|
||||
this.handleUnhealthy();
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle IPC disconnection.
|
||||
*/
|
||||
private handleDisconnection(): void {
|
||||
if (this.status !== "stopping" && this.status !== "stopped") {
|
||||
runtimeLog.error("IPC channel disconnected unexpectedly");
|
||||
this.handleUnhealthy();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update status and emit health-changed event.
|
||||
*/
|
||||
private setStatus(newStatus: RuntimeStatus): void {
|
||||
const previous = this.status;
|
||||
this.status = newStatus;
|
||||
|
||||
if (previous !== newStatus) {
|
||||
this.emit("health-changed", { status: newStatus, previous });
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user