feat(KB-616): add ProjectRuntime abstraction for task execution isolation
- Define ProjectRuntime interface with unified task execution contract\n- Implement IPC protocol for host-worker communication (messages, streaming, heartbeats)\n- Add InProcessRuntime for synchronous in-process task execution\n- Add ChildProcessRuntime with isolated worker processes for sandboxed execution\n- Implement ProjectManager to coordinate runtime selection and task lifecycle\n- Update engine exports to expose runtime APIs\n- Add comprehensive tests for all runtime implementations and IPC protocol
This commit is contained in:
102
packages/engine/src/runtimes/child-process-runtime.test.ts
Normal file
102
packages/engine/src/runtimes/child-process-runtime.test.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import type { CentralCore } from "@fusion/core";
|
||||
import { ChildProcessRuntime } from "./child-process-runtime.js";
|
||||
import type { ProjectRuntimeConfig } from "../project-runtime.js";
|
||||
|
||||
// Mock child_process
|
||||
vi.mock("node:child_process", () => ({
|
||||
fork: vi.fn().mockReturnValue({
|
||||
on: vi.fn(),
|
||||
kill: vi.fn(),
|
||||
killed: false,
|
||||
connected: false,
|
||||
send: vi.fn(),
|
||||
}),
|
||||
}));
|
||||
|
||||
describe("ChildProcessRuntime", () => {
|
||||
let runtime: ChildProcessRuntime;
|
||||
let mockCentralCore: CentralCore;
|
||||
const testConfig: ProjectRuntimeConfig = {
|
||||
projectId: "proj_test123",
|
||||
workingDirectory: "/tmp/test-project",
|
||||
isolationMode: "child-process",
|
||||
maxConcurrent: 2,
|
||||
maxWorktrees: 4,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
mockCentralCore = {
|
||||
getGlobalConcurrencyState: vi.fn().mockResolvedValue({
|
||||
globalMaxConcurrent: 4,
|
||||
currentlyActive: 0,
|
||||
queuedCount: 0,
|
||||
projectsActive: {},
|
||||
}),
|
||||
} as unknown as CentralCore;
|
||||
|
||||
runtime = new ChildProcessRuntime(testConfig, mockCentralCore);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await runtime.stop();
|
||||
} catch {
|
||||
// Ignore errors during cleanup
|
||||
}
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("lifecycle", () => {
|
||||
it("should start with status 'stopped'", () => {
|
||||
expect(runtime.getStatus()).toBe("stopped");
|
||||
});
|
||||
|
||||
it("should throw when getting TaskStore", () => {
|
||||
expect(() => runtime.getTaskStore()).toThrow("not accessible in ChildProcessRuntime");
|
||||
});
|
||||
|
||||
it("should throw when getting Scheduler", () => {
|
||||
expect(() => runtime.getScheduler()).toThrow("not accessible in ChildProcessRuntime");
|
||||
});
|
||||
|
||||
it("should return metrics even when stopped", () => {
|
||||
const metrics = runtime.getMetrics();
|
||||
expect(metrics.inFlightTasks).toBe(0);
|
||||
expect(metrics.activeAgents).toBe(0);
|
||||
expect(metrics.lastActivityAt).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("configuration", () => {
|
||||
it("should store projectId in config", () => {
|
||||
expect(testConfig.projectId).toBe("proj_test123");
|
||||
});
|
||||
|
||||
it("should store workingDirectory in config", () => {
|
||||
expect(testConfig.workingDirectory).toBe("/tmp/test-project");
|
||||
});
|
||||
|
||||
it("should have child-process isolation mode", () => {
|
||||
expect(testConfig.isolationMode).toBe("child-process");
|
||||
});
|
||||
});
|
||||
|
||||
describe("event handling", () => {
|
||||
it("should support health-changed event", () => {
|
||||
const handler = vi.fn();
|
||||
runtime.on("health-changed", handler);
|
||||
|
||||
// The constructor may emit health-changed, so we just verify
|
||||
// the event listener can be registered
|
||||
expect(handler).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should support error event", () => {
|
||||
const handler = vi.fn();
|
||||
runtime.on("error", handler);
|
||||
|
||||
expect(handler).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
499
packages/engine/src/runtimes/child-process-runtime.ts
Normal file
499
packages/engine/src/runtimes/child-process-runtime.ts
Normal file
@@ -0,0 +1,499 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { fork, type ChildProcess } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { dirname, join } from "node:path";
|
||||
import type {
|
||||
Task,
|
||||
TaskStore,
|
||||
CentralCore,
|
||||
} from "@fusion/core";
|
||||
import type { Scheduler } from "../scheduler.js";
|
||||
import type {
|
||||
ProjectRuntime,
|
||||
ProjectRuntimeConfig,
|
||||
RuntimeStatus,
|
||||
RuntimeMetrics,
|
||||
ProjectRuntimeEvents,
|
||||
} from "../project-runtime.js";
|
||||
import { IpcHost } from "../ipc/ipc-host.js";
|
||||
import {
|
||||
START_RUNTIME,
|
||||
STOP_RUNTIME,
|
||||
GET_STATUS,
|
||||
GET_METRICS,
|
||||
TASK_CREATED,
|
||||
TASK_MOVED,
|
||||
TASK_UPDATED,
|
||||
ERROR_EVENT,
|
||||
HEALTH_CHANGED,
|
||||
type TaskCreatedPayload,
|
||||
type TaskMovedPayload,
|
||||
type TaskUpdatedPayload,
|
||||
type ErrorEventPayload,
|
||||
type HealthChangedPayload,
|
||||
} from "../ipc/ipc-protocol.js";
|
||||
import { runtimeLog } from "../logger.js";
|
||||
|
||||
/**
|
||||
* Health monitor for tracking child process health.
|
||||
*/
|
||||
class HealthMonitor {
|
||||
private running = false;
|
||||
private missedHeartbeats = 0;
|
||||
private interval: ReturnType<typeof setInterval> | null = null;
|
||||
private restartAttempts = 0;
|
||||
private restartDelays = [1000, 5000, 15000]; // Exponential backoff: 1s, 5s, 15s
|
||||
|
||||
constructor(
|
||||
private onHealthCheck: () => Promise<boolean>,
|
||||
private onUnhealthy: () => void,
|
||||
private options: {
|
||||
intervalMs?: number;
|
||||
maxMissedHeartbeats?: number;
|
||||
maxRestartAttempts?: number;
|
||||
} = {}
|
||||
) {}
|
||||
|
||||
start(): void {
|
||||
if (this.running) return;
|
||||
this.running = true;
|
||||
|
||||
const intervalMs = this.options.intervalMs ?? 5000;
|
||||
const maxMissed = this.options.maxMissedHeartbeats ?? 3;
|
||||
|
||||
this.interval = setInterval(async () => {
|
||||
const healthy = await this.onHealthCheck();
|
||||
|
||||
if (healthy) {
|
||||
if (this.missedHeartbeats > 0) {
|
||||
runtimeLog.log(`Health recovered after ${this.missedHeartbeats} missed heartbeats`);
|
||||
}
|
||||
this.missedHeartbeats = 0;
|
||||
this.restartAttempts = 0; // Reset restart attempts on success
|
||||
} else {
|
||||
this.missedHeartbeats++;
|
||||
runtimeLog.warn(`Missed heartbeat ${this.missedHeartbeats}/${maxMissed}`);
|
||||
|
||||
if (this.missedHeartbeats >= maxMissed) {
|
||||
runtimeLog.error(`Health check failed after ${maxMissed} attempts`);
|
||||
this.onUnhealthy();
|
||||
}
|
||||
}
|
||||
}, intervalMs);
|
||||
|
||||
runtimeLog.log(`Health monitor started (interval: ${intervalMs}ms)`);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.running = false;
|
||||
if (this.interval) {
|
||||
clearInterval(this.interval);
|
||||
this.interval = null;
|
||||
}
|
||||
this.missedHeartbeats = 0;
|
||||
runtimeLog.log("Health monitor stopped");
|
||||
}
|
||||
|
||||
getRestartDelay(): number {
|
||||
const delay = this.restartDelays[this.restartAttempts] ?? this.restartDelays[this.restartDelays.length - 1];
|
||||
return delay;
|
||||
}
|
||||
|
||||
incrementRestartAttempts(): void {
|
||||
this.restartAttempts++;
|
||||
}
|
||||
|
||||
getRestartAttempts(): number {
|
||||
return this.restartAttempts;
|
||||
}
|
||||
|
||||
getMissedHeartbeats(): number {
|
||||
return this.missedHeartbeats;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* ChildProcessRuntime runs a project in an isolated child process.
|
||||
*
|
||||
* This provides stronger isolation between projects at the cost of
|
||||
* IPC overhead. The child process runs an InProcessRuntime internally
|
||||
* and communicates with the host via IPC messages.
|
||||
*
|
||||
* Features:
|
||||
* - Process isolation (separate memory space)
|
||||
* - Automatic restart on crash with exponential backoff
|
||||
* - Health monitoring via heartbeat protocol
|
||||
* - Graceful shutdown with configurable timeout
|
||||
* - Event forwarding from child process to host listeners
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const config: ProjectRuntimeConfig = {
|
||||
* projectId: "proj_abc123",
|
||||
* workingDirectory: "/path/to/project",
|
||||
* isolationMode: "child-process",
|
||||
* maxConcurrent: 2,
|
||||
* maxWorktrees: 4,
|
||||
* };
|
||||
*
|
||||
* const runtime = new ChildProcessRuntime(config, centralCore);
|
||||
* await runtime.start();
|
||||
*
|
||||
* // Access metrics via IPC
|
||||
* const metrics = runtime.getMetrics();
|
||||
*
|
||||
* await runtime.stop();
|
||||
* ```
|
||||
*/
|
||||
export class ChildProcessRuntime
|
||||
extends EventEmitter<ProjectRuntimeEvents>
|
||||
implements ProjectRuntime
|
||||
{
|
||||
private status: RuntimeStatus = "stopped";
|
||||
private child: ChildProcess | null = null;
|
||||
private ipcHost: IpcHost | null = null;
|
||||
private healthMonitor: HealthMonitor;
|
||||
private lastMetrics: RuntimeMetrics = {
|
||||
inFlightTasks: 0,
|
||||
activeAgents: 0,
|
||||
lastActivityAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
/**
|
||||
* @param config - Runtime configuration
|
||||
* @param centralCore - CentralCore reference for global coordination
|
||||
*/
|
||||
constructor(
|
||||
private config: ProjectRuntimeConfig,
|
||||
private centralCore: CentralCore
|
||||
) {
|
||||
super();
|
||||
this.setMaxListeners(100);
|
||||
|
||||
// Initialize health monitor
|
||||
this.healthMonitor = new HealthMonitor(
|
||||
async () => this.checkHealth(),
|
||||
() => this.handleUnhealthy(),
|
||||
{ intervalMs: 5000, maxMissedHeartbeats: 3, maxRestartAttempts: 3 }
|
||||
);
|
||||
|
||||
runtimeLog.log(`Created ChildProcessRuntime for project ${config.projectId}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the runtime by spawning a child process.
|
||||
*
|
||||
* Startup sequence:
|
||||
* 1. Set status to "starting"
|
||||
* 2. Fork child process pointing to worker entry point
|
||||
* 3. Set up IPC host with the child process
|
||||
* 4. Send START_RUNTIME command with serialized config
|
||||
* 5. Wait for OK response or timeout (10s)
|
||||
* 6. Start health monitoring heartbeat
|
||||
* 7. Set status to "active"
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (this.status !== "stopped") {
|
||||
throw new Error(`Cannot start runtime: current status is ${this.status}`);
|
||||
}
|
||||
|
||||
this.setStatus("starting");
|
||||
runtimeLog.log(`Starting ChildProcessRuntime for project ${this.config.projectId}`);
|
||||
|
||||
try {
|
||||
await this.spawnChild();
|
||||
this.setStatus("active");
|
||||
runtimeLog.log(`ChildProcessRuntime started for project ${this.config.projectId}`);
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
this.setStatus("errored");
|
||||
runtimeLog.error(`Failed to start ChildProcessRuntime:`, err.message);
|
||||
this.emit("error", err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawn the child process and set up IPC.
|
||||
*/
|
||||
private async spawnChild(): Promise<void> {
|
||||
// Determine worker entry point
|
||||
const workerPath = this.getWorkerPath();
|
||||
|
||||
runtimeLog.log(`Forking child process: ${workerPath}`);
|
||||
|
||||
// Fork child process
|
||||
this.child = fork(workerPath, [], {
|
||||
silent: true, // Pipe stdout/stderr
|
||||
execArgv: [], // Don't inherit exec arguments
|
||||
});
|
||||
|
||||
// Set up IPC host
|
||||
this.ipcHost = new IpcHost(this.child, { commandTimeoutMs: 10000 });
|
||||
|
||||
// Set up event forwarding
|
||||
this.setupEventForwarding();
|
||||
|
||||
// Send START_RUNTIME command
|
||||
runtimeLog.log("Sending START_RUNTIME command to child");
|
||||
await this.ipcHost.sendCommand(START_RUNTIME, { config: this.config });
|
||||
|
||||
// Start health monitoring
|
||||
this.healthMonitor.start();
|
||||
|
||||
// Handle child process exit
|
||||
this.child.on("exit", (code, signal) => {
|
||||
runtimeLog.warn(`Child process exited (code: ${code}, signal: ${signal})`);
|
||||
this.handleChildExit(code, signal);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path to the worker entry point.
|
||||
*/
|
||||
private getWorkerPath(): string {
|
||||
// In production, use the compiled .js file
|
||||
// In development/tests, use .ts with tsx
|
||||
const isCompiled = !import.meta.url.endsWith(".ts");
|
||||
|
||||
const currentDir = dirname(fileURLToPath(import.meta.url));
|
||||
const workerFile = isCompiled ? "child-process-worker.js" : "child-process-worker.ts";
|
||||
|
||||
return join(currentDir, workerFile);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up event forwarding from IPC host to runtime listeners.
|
||||
*/
|
||||
private setupEventForwarding(): void {
|
||||
if (!this.ipcHost) return;
|
||||
|
||||
// Forward task events
|
||||
this.ipcHost.on(TASK_CREATED, (payload: TaskCreatedPayload) => {
|
||||
this.emit("task:created", payload.task);
|
||||
});
|
||||
|
||||
this.ipcHost.on(TASK_MOVED, (payload: TaskMovedPayload) => {
|
||||
this.emit("task:moved", { task: payload.task, from: payload.from, to: payload.to });
|
||||
});
|
||||
|
||||
this.ipcHost.on(TASK_UPDATED, (payload: TaskUpdatedPayload) => {
|
||||
this.emit("task:updated", payload.task);
|
||||
});
|
||||
|
||||
// Forward error events
|
||||
this.ipcHost.on(ERROR_EVENT, (payload: ErrorEventPayload) => {
|
||||
const error = new Error(payload.message);
|
||||
if (payload.code) {
|
||||
(error as Error & { code: string }).code = payload.code;
|
||||
}
|
||||
this.emit("error", error);
|
||||
});
|
||||
|
||||
// Forward health change events
|
||||
this.ipcHost.on(HEALTH_CHANGED, (payload: HealthChangedPayload) => {
|
||||
this.status = payload.status as RuntimeStatus;
|
||||
this.emit("health-changed", { status: payload.status, previous: payload.previous });
|
||||
});
|
||||
|
||||
// Handle disconnect
|
||||
this.ipcHost.on("disconnect", () => {
|
||||
runtimeLog.warn("IPC host disconnected");
|
||||
this.handleDisconnection();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the runtime with graceful shutdown.
|
||||
*
|
||||
* Shutdown sequence:
|
||||
* 1. Set status to "stopping"
|
||||
* 2. Stop health monitoring
|
||||
* 3. Send STOP_RUNTIME command with 30s timeout
|
||||
* 4. Kill child process if graceful shutdown fails
|
||||
* 5. Set status to "stopped"
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
if (this.status === "stopped" || this.status === "stopping") {
|
||||
return;
|
||||
}
|
||||
|
||||
this.setStatus("stopping");
|
||||
runtimeLog.log(`Stopping ChildProcessRuntime for project ${this.config.projectId}`);
|
||||
|
||||
// Stop health monitoring
|
||||
this.healthMonitor.stop();
|
||||
|
||||
try {
|
||||
// Send graceful shutdown command
|
||||
if (this.ipcHost?.isConnected()) {
|
||||
runtimeLog.log("Sending STOP_RUNTIME command to child");
|
||||
await this.ipcHost.sendCommand(STOP_RUNTIME, { timeoutMs: 30000 }, 35000);
|
||||
}
|
||||
} catch (error) {
|
||||
runtimeLog.warn(`Graceful shutdown failed: ${error}`);
|
||||
}
|
||||
|
||||
// Kill child process if still running
|
||||
this.killChild();
|
||||
|
||||
this.setStatus("stopped");
|
||||
runtimeLog.log(`ChildProcessRuntime stopped for project ${this.config.projectId}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Kill the child process forcefully.
|
||||
*/
|
||||
private killChild(): void {
|
||||
if (this.child && !this.child.killed) {
|
||||
runtimeLog.log("Killing child process");
|
||||
this.child.kill("SIGTERM");
|
||||
|
||||
// Force kill after 5 seconds if still running
|
||||
setTimeout(() => {
|
||||
if (this.child && !this.child.killed) {
|
||||
runtimeLog.warn("Force killing child process");
|
||||
this.child.kill("SIGKILL");
|
||||
}
|
||||
}, 5000);
|
||||
}
|
||||
|
||||
this.child = null;
|
||||
this.ipcHost = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current runtime status.
|
||||
*/
|
||||
getStatus(): RuntimeStatus {
|
||||
return this.status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the project's TaskStore instance.
|
||||
* @throws Error - Not accessible in child mode (use IPC instead)
|
||||
*/
|
||||
getTaskStore(): TaskStore {
|
||||
throw new Error(
|
||||
"TaskStore is not accessible in ChildProcessRuntime. " +
|
||||
"Use IPC methods to access task data."
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the project's Scheduler instance.
|
||||
* @throws Error - Not accessible in child mode
|
||||
*/
|
||||
getScheduler(): Scheduler {
|
||||
throw new Error(
|
||||
"Scheduler is not accessible in ChildProcessRuntime. " +
|
||||
"Use IPC methods to interact with the scheduler."
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current runtime metrics (via IPC query).
|
||||
*/
|
||||
getMetrics(): RuntimeMetrics {
|
||||
// Query metrics via IPC if connected
|
||||
if (this.ipcHost?.isConnected()) {
|
||||
// Fire-and-forget metrics request - returns cached value immediately
|
||||
this.ipcHost
|
||||
.sendCommand(GET_METRICS, {})
|
||||
.then((metrics: unknown) => {
|
||||
this.lastMetrics = metrics as RuntimeMetrics;
|
||||
})
|
||||
.catch(() => {
|
||||
// Ignore errors, use cached value
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...this.lastMetrics,
|
||||
lastActivityAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check health by pinging the child process.
|
||||
*/
|
||||
private async checkHealth(): Promise<boolean> {
|
||||
if (!this.ipcHost?.isConnected()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.ipcHost.ping(5000);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle unhealthy child process (restart or error).
|
||||
*/
|
||||
private handleUnhealthy(): void {
|
||||
const maxRestarts = 3;
|
||||
|
||||
if (this.healthMonitor.getRestartAttempts() >= maxRestarts) {
|
||||
runtimeLog.error(`Max restart attempts (${maxRestarts}) reached, transitioning to errored`);
|
||||
this.setStatus("errored");
|
||||
this.emit("error", new Error("Child process failed after max restart attempts"));
|
||||
return;
|
||||
}
|
||||
|
||||
const delay = this.healthMonitor.getRestartDelay();
|
||||
this.healthMonitor.incrementRestartAttempts();
|
||||
|
||||
runtimeLog.log(`Attempting restart ${this.healthMonitor.getRestartAttempts()}/${maxRestarts} after ${delay}ms`);
|
||||
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
this.killChild();
|
||||
await this.spawnChild();
|
||||
runtimeLog.log("Child process restarted successfully");
|
||||
} catch (error) {
|
||||
runtimeLog.error("Failed to restart child process:", error);
|
||||
this.setStatus("errored");
|
||||
this.emit("error", error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
}, delay);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle child process exit.
|
||||
*/
|
||||
private handleChildExit(code: number | null, signal: string | null): void {
|
||||
// Don't restart if we're intentionally stopping
|
||||
if (this.status === "stopping" || this.status === "stopped") {
|
||||
return;
|
||||
}
|
||||
|
||||
// Unexpected exit - trigger restart
|
||||
runtimeLog.warn(`Unexpected child exit (code: ${code}, signal: ${signal})`);
|
||||
this.handleUnhealthy();
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle IPC disconnection.
|
||||
*/
|
||||
private handleDisconnection(): void {
|
||||
if (this.status !== "stopping" && this.status !== "stopped") {
|
||||
runtimeLog.error("IPC channel disconnected unexpectedly");
|
||||
this.handleUnhealthy();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update status and emit health-changed event.
|
||||
*/
|
||||
private setStatus(newStatus: RuntimeStatus): void {
|
||||
const previous = this.status;
|
||||
this.status = newStatus;
|
||||
|
||||
if (previous !== newStatus) {
|
||||
this.emit("health-changed", { status: newStatus, previous });
|
||||
}
|
||||
}
|
||||
}
|
||||
174
packages/engine/src/runtimes/child-process-worker.ts
Normal file
174
packages/engine/src/runtimes/child-process-worker.ts
Normal file
@@ -0,0 +1,174 @@
|
||||
/**
|
||||
* Child Process Worker Entry Point
|
||||
*
|
||||
* This module runs inside a forked child process and creates an InProcessRuntime
|
||||
* internally. It communicates with the host via IPC using the IpcWorker class.
|
||||
*
|
||||
* The worker:
|
||||
* 1. Detects if it's running as a forked child (process.send available)
|
||||
* 2. Creates an IpcWorker instance
|
||||
* 3. Registers command handlers (START_RUNTIME, STOP_RUNTIME, etc.)
|
||||
* 4. Forwards all runtime events to the host via IPC
|
||||
* 5. Handles graceful shutdown on SIGTERM
|
||||
*/
|
||||
|
||||
import { IpcWorker } from "../ipc/ipc-worker.js";
|
||||
import {
|
||||
START_RUNTIME,
|
||||
STOP_RUNTIME,
|
||||
GET_STATUS,
|
||||
GET_METRICS,
|
||||
ERROR_EVENT,
|
||||
type StartRuntimePayload,
|
||||
type StopRuntimePayload,
|
||||
} from "../ipc/ipc-protocol.js";
|
||||
import { InProcessRuntime } from "./in-process-runtime.js";
|
||||
import type { ProjectRuntimeConfig } from "../project-runtime.js";
|
||||
import { runtimeLog } from "../logger.js";
|
||||
import { CentralCore } from "@fusion/core";
|
||||
|
||||
// Only run if we're in a forked child process
|
||||
if (!process.send) {
|
||||
console.error("This module must be run as a forked child process");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
runtimeLog.log("Child process worker starting...");
|
||||
|
||||
// Create IPC worker
|
||||
const ipcWorker = new IpcWorker();
|
||||
|
||||
// InProcessRuntime instance (created when START_RUNTIME is received)
|
||||
let runtime: InProcessRuntime | null = null;
|
||||
|
||||
// Create a minimal CentralCore stub for the child process
|
||||
// The child doesn't need full CentralCore functionality
|
||||
const createStubCentralCore = (): CentralCore => {
|
||||
return {
|
||||
getGlobalConcurrencyState: async () => ({
|
||||
globalMaxConcurrent: 4,
|
||||
currentlyActive: 0,
|
||||
queuedCount: 0,
|
||||
projectsActive: {},
|
||||
}),
|
||||
recordTaskCompletion: async () => {},
|
||||
} as unknown as CentralCore;
|
||||
};
|
||||
|
||||
// Register command handlers
|
||||
|
||||
// START_RUNTIME: Create and start the InProcessRuntime
|
||||
ipcWorker.onCommand(START_RUNTIME, async (payload: unknown) => {
|
||||
const { config } = payload as StartRuntimePayload;
|
||||
runtimeLog.log(`Received START_RUNTIME command for project ${config.projectId}`);
|
||||
|
||||
if (runtime) {
|
||||
throw new Error("Runtime already started");
|
||||
}
|
||||
|
||||
// Create stub CentralCore (real coordination happens in host)
|
||||
const centralCore = createStubCentralCore();
|
||||
|
||||
// Create InProcessRuntime
|
||||
runtime = new InProcessRuntime(config, centralCore);
|
||||
|
||||
// Forward runtime events to host
|
||||
runtime.on("task:created", (task) => {
|
||||
ipcWorker.sendEvent("TASK_CREATED", { task });
|
||||
});
|
||||
|
||||
runtime.on("task:moved", (data) => {
|
||||
ipcWorker.sendEvent("TASK_MOVED", data);
|
||||
});
|
||||
|
||||
runtime.on("task:updated", (task) => {
|
||||
ipcWorker.sendEvent("TASK_UPDATED", { task });
|
||||
});
|
||||
|
||||
runtime.on("error", (error) => {
|
||||
ipcWorker.sendEvent(ERROR_EVENT, {
|
||||
message: error.message,
|
||||
code: (error as Error & { code?: string }).code,
|
||||
});
|
||||
});
|
||||
|
||||
runtime.on("health-changed", (data) => {
|
||||
ipcWorker.sendEvent("HEALTH_CHANGED", data);
|
||||
});
|
||||
|
||||
// Start the runtime
|
||||
await runtime.start();
|
||||
|
||||
runtimeLog.log("Runtime started successfully");
|
||||
return { status: runtime.getStatus() };
|
||||
});
|
||||
|
||||
// STOP_RUNTIME: Stop the runtime gracefully
|
||||
ipcWorker.onCommand(STOP_RUNTIME, async (payload: unknown) => {
|
||||
runtimeLog.log("Received STOP_RUNTIME command");
|
||||
|
||||
if (!runtime) {
|
||||
throw new Error("Runtime not started");
|
||||
}
|
||||
|
||||
const { timeoutMs } = (payload as StopRuntimePayload) || {};
|
||||
|
||||
await runtime.stop();
|
||||
runtime = null;
|
||||
|
||||
runtimeLog.log("Runtime stopped successfully");
|
||||
return { stopped: true };
|
||||
});
|
||||
|
||||
// GET_STATUS: Return current runtime status
|
||||
ipcWorker.onCommand(GET_STATUS, async () => {
|
||||
if (!runtime) {
|
||||
return { status: "stopped" };
|
||||
}
|
||||
return { status: runtime.getStatus() };
|
||||
});
|
||||
|
||||
// GET_METRICS: Return runtime metrics
|
||||
ipcWorker.onCommand(GET_METRICS, async () => {
|
||||
if (!runtime) {
|
||||
return {
|
||||
inFlightTasks: 0,
|
||||
activeAgents: 0,
|
||||
lastActivityAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
return runtime.getMetrics();
|
||||
});
|
||||
|
||||
// Handle graceful shutdown
|
||||
process.on("SIGTERM", async () => {
|
||||
runtimeLog.log("Received SIGTERM, initiating graceful shutdown...");
|
||||
|
||||
if (runtime) {
|
||||
try {
|
||||
await runtime.stop();
|
||||
runtimeLog.log("Runtime stopped gracefully");
|
||||
} catch (error) {
|
||||
runtimeLog.error("Error during graceful shutdown:", error);
|
||||
}
|
||||
}
|
||||
|
||||
ipcWorker.shutdown();
|
||||
});
|
||||
|
||||
process.on("SIGINT", async () => {
|
||||
runtimeLog.log("Received SIGINT, initiating graceful shutdown...");
|
||||
|
||||
if (runtime) {
|
||||
try {
|
||||
await runtime.stop();
|
||||
runtimeLog.log("Runtime stopped gracefully");
|
||||
} catch (error) {
|
||||
runtimeLog.error("Error during graceful shutdown:", error);
|
||||
}
|
||||
}
|
||||
|
||||
ipcWorker.shutdown();
|
||||
});
|
||||
|
||||
runtimeLog.log("Child process worker initialized and ready");
|
||||
268
packages/engine/src/runtimes/in-process-runtime.test.ts
Normal file
268
packages/engine/src/runtimes/in-process-runtime.test.ts
Normal file
@@ -0,0 +1,268 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { EventEmitter } from "node:events";
|
||||
import type { Task, TaskStore, CentralCore } from "@fusion/core";
|
||||
import { InProcessRuntime } from "./in-process-runtime.js";
|
||||
import type { ProjectRuntimeConfig } from "../project-runtime.js";
|
||||
|
||||
// Mock the TaskStore class
|
||||
vi.mock("@fusion/core", async () => {
|
||||
const actual = await vi.importActual<typeof import("@fusion/core")>("@fusion/core");
|
||||
|
||||
return {
|
||||
...actual,
|
||||
TaskStore: vi.fn().mockImplementation(function(this: TaskStore, rootDir: string) {
|
||||
const self = this as unknown as Record<string, unknown>;
|
||||
self.getRootDir = () => rootDir;
|
||||
self.init = vi.fn().mockResolvedValue(undefined);
|
||||
self.listTasks = vi.fn().mockResolvedValue([]);
|
||||
self.getSettings = vi.fn().mockResolvedValue({});
|
||||
self.on = vi.fn().mockReturnValue(self);
|
||||
self.emit = vi.fn().mockReturnValue(true);
|
||||
return self;
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
// Mock the worktree pool
|
||||
vi.mock("../worktree-pool.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../worktree-pool.js")>("../worktree-pool.js");
|
||||
|
||||
return {
|
||||
...actual,
|
||||
scanIdleWorktrees: vi.fn().mockResolvedValue([]),
|
||||
};
|
||||
});
|
||||
|
||||
// Mock the scheduler
|
||||
vi.mock("../scheduler.js", async () => {
|
||||
return {
|
||||
Scheduler: vi.fn().mockImplementation(() => {
|
||||
const self = {} as Record<string, unknown>;
|
||||
self.start = vi.fn();
|
||||
self.stop = vi.fn();
|
||||
return self;
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
// Mock the executor
|
||||
vi.mock("../executor.js", async () => {
|
||||
return {
|
||||
TaskExecutor: vi.fn().mockImplementation(() => {
|
||||
const self = {} as Record<string, unknown>;
|
||||
self.resumeOrphaned = vi.fn().mockResolvedValue(undefined);
|
||||
self.activeWorktrees = new Map();
|
||||
return self;
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
describe("InProcessRuntime", () => {
|
||||
let runtime: InProcessRuntime;
|
||||
let mockCentralCore: CentralCore;
|
||||
const testConfig: ProjectRuntimeConfig = {
|
||||
projectId: "proj_test123",
|
||||
workingDirectory: "/tmp/test-project",
|
||||
isolationMode: "in-process",
|
||||
maxConcurrent: 2,
|
||||
maxWorktrees: 4,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
// Create mock CentralCore
|
||||
mockCentralCore = {
|
||||
getGlobalConcurrencyState: vi.fn().mockResolvedValue({
|
||||
globalMaxConcurrent: 4,
|
||||
currentlyActive: 0,
|
||||
queuedCount: 0,
|
||||
projectsActive: {},
|
||||
}),
|
||||
recordTaskCompletion: vi.fn().mockResolvedValue(undefined),
|
||||
} as unknown as CentralCore;
|
||||
|
||||
runtime = new InProcessRuntime(testConfig, mockCentralCore);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await runtime.stop();
|
||||
} catch {
|
||||
// Ignore errors during cleanup
|
||||
}
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("lifecycle", () => {
|
||||
it("should start with status 'stopped'", () => {
|
||||
expect(runtime.getStatus()).toBe("stopped");
|
||||
});
|
||||
|
||||
it("should transition to 'active' after start", async () => {
|
||||
await runtime.start();
|
||||
expect(runtime.getStatus()).toBe("active");
|
||||
});
|
||||
|
||||
it("should transition to 'stopped' after stop", async () => {
|
||||
await runtime.start();
|
||||
await runtime.stop();
|
||||
expect(runtime.getStatus()).toBe("stopped");
|
||||
});
|
||||
|
||||
it("should throw if starting when not stopped", async () => {
|
||||
await runtime.start();
|
||||
await expect(runtime.start()).rejects.toThrow("Cannot start runtime");
|
||||
});
|
||||
|
||||
it("should handle stop when already stopped", async () => {
|
||||
// Should not throw
|
||||
await runtime.stop();
|
||||
expect(runtime.getStatus()).toBe("stopped");
|
||||
});
|
||||
|
||||
it("should transition through 'starting' during start", async () => {
|
||||
const statusChanges: string[] = [];
|
||||
runtime.on("health-changed", (data) => {
|
||||
statusChanges.push(data.status);
|
||||
});
|
||||
|
||||
await runtime.start();
|
||||
|
||||
expect(statusChanges).toContain("starting");
|
||||
expect(statusChanges).toContain("active");
|
||||
});
|
||||
|
||||
it("should transition through 'stopping' during stop", async () => {
|
||||
await runtime.start();
|
||||
|
||||
const statusChanges: string[] = [];
|
||||
runtime.on("health-changed", (data) => {
|
||||
statusChanges.push(data.status);
|
||||
});
|
||||
|
||||
await runtime.stop();
|
||||
|
||||
expect(statusChanges).toContain("stopping");
|
||||
expect(statusChanges).toContain("stopped");
|
||||
});
|
||||
});
|
||||
|
||||
describe("event forwarding", () => {
|
||||
it("should emit health-changed on status transitions", async () => {
|
||||
const healthChangedSpy = vi.fn();
|
||||
runtime.on("health-changed", healthChangedSpy);
|
||||
|
||||
await runtime.start();
|
||||
|
||||
expect(healthChangedSpy).toHaveBeenCalled();
|
||||
const calls = healthChangedSpy.mock.calls;
|
||||
const lastCall = calls[calls.length - 1][0];
|
||||
expect(lastCall.status).toBe("active");
|
||||
expect(lastCall.previous).toBe("starting");
|
||||
});
|
||||
|
||||
it("should emit task:created when task store emits task:created", async () => {
|
||||
await runtime.start();
|
||||
|
||||
const taskCreatedSpy = vi.fn();
|
||||
runtime.on("task:created", taskCreatedSpy);
|
||||
|
||||
// Get the mock TaskStore and simulate an event
|
||||
const taskStore = runtime.getTaskStore();
|
||||
const mockTask = { id: "KB-001", title: "Test Task" } as Task;
|
||||
|
||||
// Get the registered handler and call it
|
||||
const onCalls = (taskStore.on as ReturnType<typeof vi.fn>).mock.calls;
|
||||
const taskCreatedHandler = onCalls.find((call: unknown[]) => call[0] === "task:created");
|
||||
|
||||
if (taskCreatedHandler) {
|
||||
(taskCreatedHandler[1] as (task: Task) => void)(mockTask);
|
||||
}
|
||||
|
||||
expect(taskCreatedSpy).toHaveBeenCalledWith(mockTask);
|
||||
});
|
||||
|
||||
it("should emit task:moved when task store emits task:moved", async () => {
|
||||
await runtime.start();
|
||||
|
||||
const taskMovedSpy = vi.fn();
|
||||
runtime.on("task:moved", taskMovedSpy);
|
||||
|
||||
const taskStore = runtime.getTaskStore();
|
||||
const mockTask = { id: "KB-001", title: "Test Task" } as Task;
|
||||
const moveData = { task: mockTask, from: "todo", to: "in-progress" };
|
||||
|
||||
const onCalls = (taskStore.on as ReturnType<typeof vi.fn>).mock.calls;
|
||||
const taskMovedHandler = onCalls.find((call: unknown[]) => call[0] === "task:moved");
|
||||
|
||||
if (taskMovedHandler) {
|
||||
(taskMovedHandler[1] as (data: { task: Task; from: string; to: string }) => void)(moveData);
|
||||
}
|
||||
|
||||
expect(taskMovedSpy).toHaveBeenCalledWith(moveData);
|
||||
});
|
||||
});
|
||||
|
||||
describe("metrics", () => {
|
||||
it("should return metrics with default values before start", () => {
|
||||
const metrics = runtime.getMetrics();
|
||||
|
||||
expect(metrics.inFlightTasks).toBe(0);
|
||||
expect(metrics.activeAgents).toBe(0);
|
||||
expect(metrics.lastActivityAt).toBeDefined();
|
||||
});
|
||||
|
||||
it("should include memory usage in metrics", () => {
|
||||
const metrics = runtime.getMetrics();
|
||||
|
||||
// Memory usage may or may not be available depending on environment
|
||||
if (metrics.memoryBytes !== undefined) {
|
||||
expect(typeof metrics.memoryBytes).toBe("number");
|
||||
expect(metrics.memoryBytes).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("accessors", () => {
|
||||
it("should throw when accessing TaskStore before start", () => {
|
||||
expect(() => runtime.getTaskStore()).toThrow("TaskStore not initialized");
|
||||
});
|
||||
|
||||
it("should throw when accessing Scheduler before start", () => {
|
||||
expect(() => runtime.getScheduler()).toThrow("Scheduler not initialized");
|
||||
});
|
||||
|
||||
it("should return TaskStore after start", async () => {
|
||||
await runtime.start();
|
||||
const taskStore = runtime.getTaskStore();
|
||||
|
||||
expect(taskStore).toBeDefined();
|
||||
expect(taskStore.getRootDir()).toBe(testConfig.workingDirectory);
|
||||
});
|
||||
|
||||
it("should return Scheduler after start", async () => {
|
||||
await runtime.start();
|
||||
const scheduler = runtime.getScheduler();
|
||||
|
||||
expect(scheduler).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("configuration", () => {
|
||||
it("should store projectId in config", () => {
|
||||
// Access via the constructor params - runtime is created with testConfig
|
||||
expect(testConfig.projectId).toBe("proj_test123");
|
||||
});
|
||||
|
||||
it("should store workingDirectory in config", () => {
|
||||
expect(testConfig.workingDirectory).toBe("/tmp/test-project");
|
||||
});
|
||||
|
||||
it("should store maxConcurrent in config", () => {
|
||||
expect(testConfig.maxConcurrent).toBe(2);
|
||||
});
|
||||
|
||||
it("should store maxWorktrees in config", () => {
|
||||
expect(testConfig.maxWorktrees).toBe(4);
|
||||
});
|
||||
});
|
||||
});
|
||||
395
packages/engine/src/runtimes/in-process-runtime.ts
Normal file
395
packages/engine/src/runtimes/in-process-runtime.ts
Normal file
@@ -0,0 +1,395 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import type {
|
||||
TaskStore,
|
||||
Task,
|
||||
CentralCore,
|
||||
} from "@fusion/core";
|
||||
import { Scheduler } from "../scheduler.js";
|
||||
import { TaskExecutor, type TaskExecutorOptions } from "../executor.js";
|
||||
import { WorktreePool } from "../worktree-pool.js";
|
||||
import { AgentSemaphore } from "../concurrency.js";
|
||||
import type {
|
||||
ProjectRuntime,
|
||||
ProjectRuntimeConfig,
|
||||
RuntimeStatus,
|
||||
RuntimeMetrics,
|
||||
ProjectRuntimeEvents,
|
||||
} from "../project-runtime.js";
|
||||
import { runtimeLog } from "../logger.js";
|
||||
import type { StuckTaskDetector } from "../stuck-task-detector.js";
|
||||
import type { UsageLimitPauser } from "../usage-limit-detector.js";
|
||||
|
||||
/**
|
||||
* InProcessRuntime runs a project within the main process.
|
||||
*
|
||||
* This is the default execution mode — all components (TaskStore, Scheduler,
|
||||
* Executor, WorktreePool) share the same memory space and event loop.
|
||||
*
|
||||
* Features:
|
||||
* - Direct access to TaskStore and Scheduler via getter methods
|
||||
* - Synchronous event forwarding from TaskStore to runtime listeners
|
||||
* - Graceful shutdown with configurable timeout
|
||||
* - Automatic orphaned task recovery on startup
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const config: ProjectRuntimeConfig = {
|
||||
* projectId: "proj_abc123",
|
||||
* workingDirectory: "/path/to/project",
|
||||
* isolationMode: "in-process",
|
||||
* maxConcurrent: 2,
|
||||
* maxWorktrees: 4,
|
||||
* };
|
||||
*
|
||||
* const runtime = new InProcessRuntime(config, centralCore);
|
||||
* await runtime.start();
|
||||
*
|
||||
* // Access components directly
|
||||
* const taskStore = runtime.getTaskStore();
|
||||
* const scheduler = runtime.getScheduler();
|
||||
*
|
||||
* await runtime.stop();
|
||||
* ```
|
||||
*/
|
||||
export class InProcessRuntime
|
||||
extends EventEmitter<ProjectRuntimeEvents>
|
||||
implements ProjectRuntime
|
||||
{
|
||||
private status: RuntimeStatus = "stopped";
|
||||
private taskStore!: TaskStore;
|
||||
private scheduler!: Scheduler;
|
||||
private executor!: TaskExecutor;
|
||||
private worktreePool!: WorktreePool;
|
||||
private globalSemaphore?: AgentSemaphore;
|
||||
private stuckTaskDetector?: StuckTaskDetector;
|
||||
private usageLimitPauser?: UsageLimitPauser;
|
||||
private lastActivityAt: string = new Date().toISOString();
|
||||
|
||||
/**
|
||||
* @param config - Runtime configuration
|
||||
* @param centralCore - CentralCore reference for global coordination
|
||||
*/
|
||||
constructor(
|
||||
private config: ProjectRuntimeConfig,
|
||||
private centralCore: CentralCore
|
||||
) {
|
||||
super();
|
||||
this.setMaxListeners(100);
|
||||
runtimeLog.log(`Created InProcessRuntime for project ${config.projectId}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the runtime and initialize all subsystems.
|
||||
*
|
||||
* Initialization order:
|
||||
* 1. Initialize TaskStore
|
||||
* 2. Initialize WorktreePool
|
||||
* 3. Initialize Scheduler (with TaskStore)
|
||||
* 4. Initialize TaskExecutor (with TaskStore, worktree pool, global semaphore)
|
||||
* 5. Resume orphaned in-progress tasks
|
||||
* 6. Start scheduler
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (this.status !== "stopped") {
|
||||
throw new Error(`Cannot start runtime: current status is ${this.status}`);
|
||||
}
|
||||
|
||||
this.setStatus("starting");
|
||||
runtimeLog.log(`Starting InProcessRuntime for project ${this.config.projectId}`);
|
||||
|
||||
try {
|
||||
// 1. Initialize TaskStore
|
||||
const { TaskStore } = await import("@fusion/core");
|
||||
this.taskStore = new TaskStore(this.config.workingDirectory);
|
||||
await this.taskStore.init();
|
||||
runtimeLog.log(`TaskStore initialized for project ${this.config.projectId}`);
|
||||
|
||||
// 2. Initialize WorktreePool
|
||||
this.worktreePool = new WorktreePool();
|
||||
|
||||
// Rehydrate pool from disk state (idle worktrees)
|
||||
const { scanIdleWorktrees } = await import("../worktree-pool.js");
|
||||
const idleWorktrees = await scanIdleWorktrees(
|
||||
this.config.workingDirectory,
|
||||
this.taskStore
|
||||
);
|
||||
if (idleWorktrees.length > 0) {
|
||||
this.worktreePool.rehydrate(idleWorktrees);
|
||||
runtimeLog.log(
|
||||
`Rehydrated worktree pool with ${idleWorktrees.length} idle worktrees`
|
||||
);
|
||||
}
|
||||
|
||||
// 3. Initialize global semaphore from CentralCore
|
||||
const globalLimit = await this.getGlobalConcurrencyLimit();
|
||||
this.globalSemaphore = new AgentSemaphore(() => globalLimit);
|
||||
|
||||
// 4. Initialize Scheduler
|
||||
this.scheduler = new Scheduler(this.taskStore, {
|
||||
maxConcurrent: this.config.maxConcurrent,
|
||||
maxWorktrees: this.config.maxWorktrees,
|
||||
semaphore: this.globalSemaphore,
|
||||
onSchedule: (task) => {
|
||||
this.recordActivity();
|
||||
runtimeLog.log(`Scheduled task ${task.id}`);
|
||||
},
|
||||
onBlocked: (task, blockedBy) => {
|
||||
runtimeLog.log(`Task ${task.id} blocked by: ${blockedBy.join(", ")}`);
|
||||
},
|
||||
});
|
||||
|
||||
// 5. Initialize TaskExecutor
|
||||
const executorOptions: TaskExecutorOptions = {
|
||||
semaphore: this.globalSemaphore,
|
||||
pool: this.worktreePool,
|
||||
usageLimitPauser: this.usageLimitPauser,
|
||||
stuckTaskDetector: this.stuckTaskDetector,
|
||||
onStart: (task, worktreePath) => {
|
||||
this.recordActivity();
|
||||
runtimeLog.log(`Started executing task ${task.id} in ${worktreePath}`);
|
||||
},
|
||||
onComplete: (task) => {
|
||||
this.recordActivity();
|
||||
runtimeLog.log(`Completed task ${task.id}`);
|
||||
// Record task completion in CentralCore
|
||||
this.recordTaskCompletion(task.id, true);
|
||||
},
|
||||
onError: (task, error) => {
|
||||
this.recordActivity();
|
||||
runtimeLog.error(`Task ${task.id} failed:`, error.message);
|
||||
this.recordTaskCompletion(task.id, false);
|
||||
},
|
||||
};
|
||||
|
||||
this.executor = new TaskExecutor(
|
||||
this.taskStore,
|
||||
this.config.workingDirectory,
|
||||
executorOptions
|
||||
);
|
||||
|
||||
// 6. Set up event forwarding from TaskStore
|
||||
this.setupEventForwarding();
|
||||
|
||||
// 7. Resume orphaned in-progress tasks
|
||||
await this.executor.resumeOrphaned();
|
||||
|
||||
// 8. Start scheduler
|
||||
this.scheduler.start();
|
||||
|
||||
this.setStatus("active");
|
||||
runtimeLog.log(`InProcessRuntime started for project ${this.config.projectId}`);
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
this.setStatus("errored");
|
||||
runtimeLog.error(`Failed to start InProcessRuntime:`, err.message);
|
||||
this.emit("error", err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the runtime with graceful shutdown.
|
||||
*
|
||||
* Shutdown sequence:
|
||||
* 1. Set status to "stopping"
|
||||
* 2. Stop scheduler (no new tasks)
|
||||
* 3. Wait for executor to finish active tasks (with timeout)
|
||||
* 4. Drain and cleanup worktree pool
|
||||
* 5. Set status to "stopped"
|
||||
*
|
||||
* @throws Error if shutdown timeout is exceeded
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
if (this.status === "stopped" || this.status === "stopping") {
|
||||
return;
|
||||
}
|
||||
|
||||
this.setStatus("stopping");
|
||||
runtimeLog.log(`Stopping InProcessRuntime for project ${this.config.projectId}`);
|
||||
|
||||
try {
|
||||
// 1. Stop scheduler (prevents new task scheduling)
|
||||
if (this.scheduler) {
|
||||
this.scheduler.stop();
|
||||
runtimeLog.log("Scheduler stopped");
|
||||
}
|
||||
|
||||
// 2. Wait for active tasks to complete (30 second timeout)
|
||||
const shutdownTimeout = 30000;
|
||||
const startTime = Date.now();
|
||||
|
||||
while (Date.now() - startTime < shutdownTimeout) {
|
||||
const metrics = this.getMetrics();
|
||||
if (metrics.inFlightTasks === 0) {
|
||||
break;
|
||||
}
|
||||
runtimeLog.log(
|
||||
`Waiting for ${metrics.inFlightTasks} in-flight tasks to complete...`
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
|
||||
// Check if we timed out
|
||||
const finalMetrics = this.getMetrics();
|
||||
if (finalMetrics.inFlightTasks > 0) {
|
||||
runtimeLog.warn(
|
||||
`Shutdown timeout reached with ${finalMetrics.inFlightTasks} tasks still in-flight`
|
||||
);
|
||||
}
|
||||
|
||||
// 3. Drain and cleanup worktree pool
|
||||
if (this.worktreePool) {
|
||||
const worktrees = this.worktreePool.drain();
|
||||
if (worktrees.length > 0) {
|
||||
runtimeLog.log(`Drained ${worktrees.length} worktrees from pool`);
|
||||
}
|
||||
}
|
||||
|
||||
this.setStatus("stopped");
|
||||
runtimeLog.log(`InProcessRuntime stopped for project ${this.config.projectId}`);
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
this.setStatus("errored");
|
||||
runtimeLog.error(`Error during shutdown:`, err.message);
|
||||
this.emit("error", err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current runtime status.
|
||||
*/
|
||||
getStatus(): RuntimeStatus {
|
||||
return this.status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the project's TaskStore instance.
|
||||
* @throws Error if runtime has not been started
|
||||
*/
|
||||
getTaskStore(): TaskStore {
|
||||
if (!this.taskStore) {
|
||||
throw new Error("TaskStore not initialized. Call start() first.");
|
||||
}
|
||||
return this.taskStore;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the project's Scheduler instance.
|
||||
* @throws Error if runtime has not been started
|
||||
*/
|
||||
getScheduler(): Scheduler {
|
||||
if (!this.scheduler) {
|
||||
throw new Error("Scheduler not initialized. Call start() first.");
|
||||
}
|
||||
return this.scheduler;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current runtime metrics.
|
||||
*/
|
||||
getMetrics(): RuntimeMetrics {
|
||||
// Estimate in-flight tasks by checking active sessions
|
||||
const inFlightTasks = this.executor
|
||||
? (this.executor as unknown as { activeWorktrees?: Map<string, string> }).activeWorktrees?.size ?? 0
|
||||
: 0;
|
||||
|
||||
// Get active agent count from the semaphore
|
||||
const activeAgents = this.globalSemaphore?.activeCount ?? 0;
|
||||
|
||||
// Get memory usage if available
|
||||
const memoryBytes = process.memoryUsage?.().heapUsed;
|
||||
|
||||
return {
|
||||
inFlightTasks,
|
||||
activeAgents,
|
||||
lastActivityAt: this.lastActivityAt,
|
||||
memoryBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the StuckTaskDetector for this runtime.
|
||||
*/
|
||||
setStuckTaskDetector(detector: StuckTaskDetector): void {
|
||||
this.stuckTaskDetector = detector;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the UsageLimitPauser for this runtime.
|
||||
*/
|
||||
setUsageLimitPauser(pauser: UsageLimitPauser): void {
|
||||
this.usageLimitPauser = pauser;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up event forwarding from TaskStore to runtime listeners.
|
||||
*/
|
||||
private setupEventForwarding(): void {
|
||||
// Forward task:created events
|
||||
this.taskStore.on("task:created", (task: Task) => {
|
||||
this.recordActivity();
|
||||
this.emit("task:created", task);
|
||||
});
|
||||
|
||||
// Forward task:moved events
|
||||
this.taskStore.on("task:moved", (data: { task: Task; from: string; to: string }) => {
|
||||
this.recordActivity();
|
||||
this.emit("task:moved", data);
|
||||
});
|
||||
|
||||
// Forward task:updated events
|
||||
this.taskStore.on("task:updated", (task: Task) => {
|
||||
this.recordActivity();
|
||||
this.emit("task:updated", task);
|
||||
});
|
||||
|
||||
runtimeLog.log("Event forwarding setup complete");
|
||||
}
|
||||
|
||||
/**
|
||||
* Update status and emit health-changed event.
|
||||
*/
|
||||
private setStatus(newStatus: RuntimeStatus): void {
|
||||
const previous = this.status;
|
||||
this.status = newStatus;
|
||||
|
||||
if (previous !== newStatus) {
|
||||
this.emit("health-changed", { status: newStatus, previous });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record activity timestamp.
|
||||
*/
|
||||
private recordActivity(): void {
|
||||
this.lastActivityAt = new Date().toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get global concurrency limit from CentralCore.
|
||||
*/
|
||||
private async getGlobalConcurrencyLimit(): Promise<number> {
|
||||
try {
|
||||
const state = await this.centralCore.getGlobalConcurrencyState();
|
||||
return state.globalMaxConcurrent;
|
||||
} catch {
|
||||
// Fallback to default if CentralCore is unavailable
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record task completion in CentralCore.
|
||||
*/
|
||||
private async recordTaskCompletion(taskId: string, success: boolean): Promise<void> {
|
||||
try {
|
||||
// Estimate duration (simplified - in reality, we'd track start time)
|
||||
const durationMs = 0; // Placeholder
|
||||
await this.centralCore.recordTaskCompletion(this.config.projectId, durationMs, success);
|
||||
} catch (error) {
|
||||
// Non-fatal: logging is best-effort
|
||||
runtimeLog.warn(`Failed to record task completion: ${error}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user