feat: multi-project engine runtime improvements
- Update project manager and runtime interfaces for multi-project coordination - Add project engine configuration to core types and settings schema - Enhance child-process worker tests for signal handling coverage - Extend in-process runtime with per-project engine lifecycle hooks Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
gsxdsm
@@ -11,43 +11,14 @@
|
||||
|
||||
import type { AddressInfo } from "node:net";
|
||||
import {
|
||||
TaskStore,
|
||||
AutomationStore,
|
||||
CentralCore,
|
||||
AgentStore,
|
||||
PluginStore,
|
||||
PluginLoader,
|
||||
getTaskMergeBlocker,
|
||||
syncInsightExtractionAutomation,
|
||||
INSIGHT_EXTRACTION_SCHEDULE_NAME,
|
||||
processAndAuditInsightExtraction,
|
||||
} from "@fusion/core";
|
||||
import type { ScheduledTask, AutomationRunResult } from "@fusion/core";
|
||||
import { createServer, GitHubClient } from "@fusion/dashboard";
|
||||
import {
|
||||
TriageProcessor,
|
||||
TaskExecutor,
|
||||
Scheduler,
|
||||
AgentSemaphore,
|
||||
WorktreePool,
|
||||
aiMergeTask,
|
||||
UsageLimitPauser,
|
||||
PRIORITY_MERGE,
|
||||
scanIdleWorktrees,
|
||||
cleanupOrphanedWorktrees,
|
||||
NtfyNotifier,
|
||||
PrMonitor,
|
||||
PrCommentHandler,
|
||||
CronRunner,
|
||||
StuckTaskDetector,
|
||||
SelfHealingManager,
|
||||
MissionAutopilot,
|
||||
MissionExecutionLoop,
|
||||
createAiPromptExecutor,
|
||||
HeartbeatMonitor,
|
||||
HeartbeatTriggerScheduler,
|
||||
type WakeContext,
|
||||
} from "@fusion/engine";
|
||||
import { ProjectEngine } from "@fusion/engine";
|
||||
import type { ProjectEngineOptions, ProjectRuntimeConfig } from "@fusion/engine";
|
||||
import {
|
||||
AuthStorage,
|
||||
DefaultPackageManager,
|
||||
@@ -233,18 +204,129 @@ export async function runServe(
|
||||
const selectedHost = opts.host ?? "0.0.0.0";
|
||||
const cwd = process.cwd();
|
||||
|
||||
const store = new TaskStore(cwd);
|
||||
await store.init();
|
||||
// ── CentralCore: global coordination + ntfy project ID lookup ─────────
|
||||
//
|
||||
// Created once and reused for:
|
||||
// 1. Looking up the registered project ID for NtfyNotifier (via ProjectEngine)
|
||||
// 2. Passed to ProjectEngine/InProcessRuntime for concurrency coordination
|
||||
// 3. Node registration for cluster awareness
|
||||
//
|
||||
let ntfyProjectId: string | undefined;
|
||||
let sharedCentralCore: CentralCore | null = null;
|
||||
try {
|
||||
sharedCentralCore = new CentralCore();
|
||||
await sharedCentralCore.init();
|
||||
const registered = await sharedCentralCore.getProjectByPath(cwd);
|
||||
if (registered) {
|
||||
ntfyProjectId = registered.id;
|
||||
}
|
||||
} catch {
|
||||
// Central DB unavailable or project not registered — backward compatible
|
||||
}
|
||||
|
||||
// ── ProjectEngine: core engine subsystems ────────────────────────────
|
||||
//
|
||||
// ProjectEngine composes InProcessRuntime with higher-level subsystems:
|
||||
// - TaskStore, Scheduler, TaskExecutor, TriageProcessor (via InProcessRuntime)
|
||||
// - WorktreePool + rehydration (via InProcessRuntime)
|
||||
// - AgentSemaphore (via InProcessRuntime)
|
||||
// - StuckTaskDetector + SelfHealingManager (via InProcessRuntime)
|
||||
// - MissionAutopilot + MissionExecutionLoop (via InProcessRuntime)
|
||||
// - PrMonitor + PrCommentHandler (via ProjectEngine)
|
||||
// - NtfyNotifier (via ProjectEngine)
|
||||
// - CronRunner + AutomationStore (via ProjectEngine)
|
||||
// - Auto-merge queue with conflict retry (via ProjectEngine)
|
||||
// - 5 settings event listeners (via ProjectEngine)
|
||||
//
|
||||
const githubClient = new GitHubClient(process.env.GITHUB_TOKEN);
|
||||
|
||||
// Post-run callback for memory insight extraction processing
|
||||
const onMemoryInsightRunProcessed = async (
|
||||
schedule: ScheduledTask,
|
||||
result: AutomationRunResult,
|
||||
): Promise<void> => {
|
||||
// Only process the memory insight extraction schedule
|
||||
if (schedule.name !== INSIGHT_EXTRACTION_SCHEDULE_NAME) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Extract the AI step output from the result
|
||||
const stepResults = result.stepResults ?? [];
|
||||
// Step name updated in FN-1477 to include pruning
|
||||
const aiStep = stepResults.find(
|
||||
(sr) => sr.stepName === "Extract Memory Insights and Prune" || sr.stepName === "Extract Memory Insights",
|
||||
);
|
||||
|
||||
if (!aiStep) {
|
||||
console.log(`[memory-audit] No insight extraction step found in ${schedule.name} result`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[memory-audit] Processing memory insight extraction run...`);
|
||||
|
||||
try {
|
||||
const auditReport = await processAndAuditInsightExtraction(cwd, {
|
||||
rawResponse: aiStep.output ?? "",
|
||||
stepSuccess: aiStep.success,
|
||||
runAt: result.startedAt,
|
||||
error: aiStep.error,
|
||||
});
|
||||
|
||||
const pruneStatus = auditReport.pruning.applied
|
||||
? ` | Pruned: ${auditReport.pruning.originalSize} → ${auditReport.pruning.newSize} chars`
|
||||
: ` | Pruning: ${auditReport.pruning.reason}`;
|
||||
|
||||
console.log(
|
||||
`[memory-audit] ✓ Audit complete — Health: ${auditReport.health}, ` +
|
||||
`Insights: ${auditReport.insightsMemory.insightCount}${pruneStatus}`,
|
||||
);
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[memory-audit] ✗ Failed to process insight extraction: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const engineOptions: ProjectEngineOptions = {
|
||||
projectId: ntfyProjectId,
|
||||
getMergeStrategy,
|
||||
processPullRequestMerge: (store, wd, taskId) =>
|
||||
processPullRequestMergeTask(store, wd, taskId, githubClient, getTaskMergeBlocker),
|
||||
getTaskMergeBlocker,
|
||||
onInsightRunProcessed: onMemoryInsightRunProcessed as any,
|
||||
};
|
||||
|
||||
const runtimeConfig: ProjectRuntimeConfig = {
|
||||
projectId: ntfyProjectId ?? cwd,
|
||||
workingDirectory: cwd,
|
||||
isolationMode: "in-process",
|
||||
// maxConcurrent/maxWorktrees are read from settings inside InProcessRuntime
|
||||
// via CentralCore; use safe defaults here.
|
||||
maxConcurrent: 4,
|
||||
maxWorktrees: 10,
|
||||
};
|
||||
|
||||
const engine = new ProjectEngine(
|
||||
runtimeConfig,
|
||||
sharedCentralCore ?? new CentralCore(),
|
||||
engineOptions,
|
||||
);
|
||||
|
||||
await engine.start();
|
||||
|
||||
const store = engine.getTaskStore();
|
||||
|
||||
// InProcessRuntime does not call store.watch() — do it here so SSE events
|
||||
// and file-watcher triggers are active for the HTTP layer.
|
||||
await store.watch();
|
||||
|
||||
// Set up database health check for diagnostics
|
||||
setServeDbHealthCheck(() => store.healthCheck());
|
||||
|
||||
const automationStore = new AutomationStore(cwd);
|
||||
await automationStore.init();
|
||||
|
||||
const agentStore = new AgentStore({ rootDir: store.getFusionDir() });
|
||||
await agentStore.init();
|
||||
if (opts.paused) {
|
||||
await store.updateSettings({ enginePaused: true });
|
||||
console.log("[engine] Starting in paused mode — automation disabled");
|
||||
}
|
||||
|
||||
// ── PluginStore: plugin installation management ─────────────────────
|
||||
//
|
||||
@@ -252,6 +334,10 @@ export async function runServe(
|
||||
// Enables the PluginManager UI to list, install, enable, disable, and
|
||||
// configure plugins via the /api/plugins REST endpoints.
|
||||
//
|
||||
// Note: InProcessRuntime creates its own PluginStore/PluginLoader/PluginRunner
|
||||
// internally for task-execution plugin hooks. These instances here serve the
|
||||
// HTTP plugin-management API routes and are intentionally separate.
|
||||
//
|
||||
const pluginStore = new PluginStore(store.getFusionDir());
|
||||
await pluginStore.init();
|
||||
|
||||
@@ -267,345 +353,15 @@ export async function runServe(
|
||||
taskStore: store,
|
||||
});
|
||||
|
||||
// ── HeartbeatMonitor: runtime monitoring (UTILITY — NO semaphore) ───
|
||||
//
|
||||
// ⚠️ UTILITY PATH: This component does NOT receive the task-lane semaphore.
|
||||
//
|
||||
// Provides the Paperclip-style heartbeat execution engine:
|
||||
// wake → check inbox → work → exit
|
||||
//
|
||||
// Enables lightweight agent sessions for monitoring, not task-lane work.
|
||||
// By design, heartbeat sessions are independent of task concurrency limits
|
||||
// so they can run regardless of how busy the task lanes are.
|
||||
//
|
||||
// Passed to createServer to enable the heartbeat routes.
|
||||
//
|
||||
let heartbeatMonitor: HeartbeatMonitor | undefined;
|
||||
let triggerScheduler: HeartbeatTriggerScheduler | undefined;
|
||||
try {
|
||||
heartbeatMonitor = new HeartbeatMonitor({
|
||||
store: agentStore,
|
||||
agentStore: agentStore, // enables per-agent config resolution
|
||||
taskStore: store,
|
||||
rootDir: cwd,
|
||||
onMissed: (agentId) => {
|
||||
console.log(`[engine] Agent ${agentId} missed heartbeat`);
|
||||
},
|
||||
onTerminated: (agentId) => {
|
||||
console.log(`[engine] Agent ${agentId} terminated (unresponsive)`);
|
||||
},
|
||||
});
|
||||
heartbeatMonitor.start();
|
||||
// Get heartbeat components from the runtime (initialized by InProcessRuntime)
|
||||
const heartbeatMonitor = engine.getRuntime().getHeartbeatMonitor();
|
||||
|
||||
// HeartbeatTriggerScheduler: trigger scheduling (UTILITY — NO semaphore) ──
|
||||
//
|
||||
// ⚠️ UTILITY PATH: This scheduler does NOT receive the task-lane semaphore.
|
||||
//
|
||||
// Manages timer and assignment-based triggers for heartbeat execution.
|
||||
// By design, trigger scheduling is independent of task-lane concurrency limits.
|
||||
//
|
||||
triggerScheduler = new HeartbeatTriggerScheduler(
|
||||
agentStore,
|
||||
async (agentId, source, context: WakeContext) => {
|
||||
if (!heartbeatMonitor) return;
|
||||
await heartbeatMonitor.executeHeartbeat({
|
||||
agentId,
|
||||
source,
|
||||
triggerDetail: context.triggerDetail,
|
||||
taskId: typeof context.taskId === "string" ? context.taskId : undefined,
|
||||
triggeringCommentIds: Array.isArray(context.triggeringCommentIds)
|
||||
? context.triggeringCommentIds.filter((id): id is string => typeof id === "string" && id.length > 0)
|
||||
: undefined,
|
||||
triggeringCommentType:
|
||||
context.triggeringCommentType === "steering"
|
||||
|| context.triggeringCommentType === "task"
|
||||
|| context.triggeringCommentType === "pr"
|
||||
? context.triggeringCommentType
|
||||
: undefined,
|
||||
contextSnapshot: { ...context },
|
||||
});
|
||||
},
|
||||
store,
|
||||
);
|
||||
triggerScheduler.start();
|
||||
// Get mission components from the runtime (initialized by InProcessRuntime)
|
||||
const missionAutopilot = engine.getRuntime().getMissionAutopilot();
|
||||
const missionExecutionLoop = engine.getRuntime().getMissionExecutionLoop();
|
||||
|
||||
// Register existing agents that have heartbeat config
|
||||
const agents = await agentStore.listAgents();
|
||||
for (const agent of agents) {
|
||||
const rc = agent.runtimeConfig;
|
||||
if (rc && (rc.heartbeatIntervalMs || rc.enabled !== undefined || rc.maxConcurrentRuns)) {
|
||||
triggerScheduler.registerAgent(agent.id, {
|
||||
heartbeatIntervalMs: rc.heartbeatIntervalMs as number | undefined,
|
||||
enabled: rc.enabled as boolean | undefined,
|
||||
maxConcurrentRuns: rc.maxConcurrentRuns as number | undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (agents.length > 0) {
|
||||
console.log(`[engine] Registered ${triggerScheduler.getRegisteredAgents().length} agents for heartbeat triggers`);
|
||||
}
|
||||
} catch (err) {
|
||||
// Non-fatal — agent monitoring is optional
|
||||
console.log(`[engine] HeartbeatMonitor initialization failed (continuing without agent monitoring):`, err);
|
||||
}
|
||||
|
||||
let ntfyProjectId: string | undefined;
|
||||
try {
|
||||
const central = new CentralCore();
|
||||
await central.init();
|
||||
const registered = await central.getProjectByPath(cwd);
|
||||
await central.close();
|
||||
if (registered) {
|
||||
ntfyProjectId = registered.id;
|
||||
}
|
||||
} catch {
|
||||
// Central DB unavailable or project not registered — backward compatible
|
||||
}
|
||||
|
||||
const notifier = new NtfyNotifier(store, { projectId: ntfyProjectId });
|
||||
notifier.start();
|
||||
|
||||
if (opts.paused) {
|
||||
await store.updateSettings({ enginePaused: true });
|
||||
console.log("[engine] Starting in paused mode — automation disabled");
|
||||
}
|
||||
|
||||
// ── Task-lane concurrency semaphore ────────────────────────────────
|
||||
//
|
||||
// ⚠️ SEMAPHORE BOUNDARY: This semaphore governs ONLY task-lane agents.
|
||||
//
|
||||
// Governed components (task lanes):
|
||||
// - TriageProcessor: specification agents that produce PROMPT.md
|
||||
// - TaskExecutor: task execution agents that implement features
|
||||
// - Scheduler: coordinates which agent gets which task
|
||||
// - onMerge: AI-powered merge execution for completed tasks
|
||||
//
|
||||
// UTILITY WORKFLOWS — NOT governed by this semaphore:
|
||||
// - HeartbeatMonitor: lightweight heartbeat sessions for agent monitoring
|
||||
// - HeartbeatTriggerScheduler: timer/assignment-based trigger scheduling
|
||||
// - CronRunner (via createAiPromptExecutor): scheduled automation prompts
|
||||
// - Model sync, auth setup, plugin loading: bootstrap/setup workflows
|
||||
//
|
||||
// This boundary prevents utility workflows from being blocked by
|
||||
// task-lane saturation and ensures utility work is always available.
|
||||
//
|
||||
// The limit is read from a cached value that is refreshed from the store
|
||||
// on each scheduler poll cycle (see engine block below). This avoids
|
||||
// async I/O in the synchronous getter while still picking up live changes.
|
||||
//
|
||||
const initialSettings = await store.getSettings();
|
||||
let cachedMaxConcurrent = initialSettings.maxConcurrent;
|
||||
const semaphore = new AgentSemaphore(() => cachedMaxConcurrent);
|
||||
|
||||
const pool = new WorktreePool();
|
||||
|
||||
if (initialSettings.recycleWorktrees) {
|
||||
const idlePaths = await scanIdleWorktrees(cwd, store);
|
||||
if (idlePaths.length > 0) {
|
||||
pool.rehydrate(idlePaths);
|
||||
console.log(`[engine] Rehydrated pool with ${idlePaths.length} idle worktree(s)`);
|
||||
}
|
||||
} else {
|
||||
const cleaned = await cleanupOrphanedWorktrees(cwd, store);
|
||||
if (cleaned > 0) {
|
||||
console.log(`[engine] Cleaned up ${cleaned} orphaned worktree(s)`);
|
||||
}
|
||||
}
|
||||
|
||||
const usageLimitPauser = new UsageLimitPauser(store);
|
||||
const githubClient = new GitHubClient(process.env.GITHUB_TOKEN);
|
||||
|
||||
// ── onMerge: AI-powered merge (TASK LANE — semaphore-gated) ─────────────
|
||||
//
|
||||
// ⚠️ TASK LANE: aiMergeTask is wrapped with semaphore.run() to ensure
|
||||
// merge agents count toward settings.maxConcurrent alongside triage and execution.
|
||||
//
|
||||
// The raw aiMergeTask does NOT receive the semaphore directly;
|
||||
// the semaphore gating is applied at the onMerge wrapper level.
|
||||
//
|
||||
// Track the active merge session so it can be killed on global pause.
|
||||
let activeMergeSession: { dispose: () => void } | null = null;
|
||||
|
||||
const rawMerge = (taskId: string) =>
|
||||
aiMergeTask(store, cwd, taskId, {
|
||||
pool,
|
||||
usageLimitPauser,
|
||||
agentStore,
|
||||
onAgentText: (delta) => process.stdout.write(delta),
|
||||
onSession: (session) => {
|
||||
activeMergeSession = session;
|
||||
},
|
||||
});
|
||||
|
||||
const onMerge = (taskId: string) =>
|
||||
semaphore.run(() => rawMerge(taskId), PRIORITY_MERGE);
|
||||
|
||||
store.on("settings:updated", ({ settings, previous }) => {
|
||||
if (settings.globalPause && !previous.globalPause) {
|
||||
if (activeMergeSession) {
|
||||
console.log("[auto-merge] Global pause — terminating active merge session");
|
||||
activeMergeSession.dispose();
|
||||
activeMergeSession = null;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const mergeQueue: string[] = [];
|
||||
const mergeActive = new Set<string>();
|
||||
let mergeRunning = false;
|
||||
const maxAutoMergeRetries = 3;
|
||||
|
||||
/**
|
||||
* Check if a task can be merged (not blocked and within retry limit).
|
||||
* This is the final validation gate before attempting a merge.
|
||||
*/
|
||||
function canMergeTask(task: { mergeRetries?: number | null; column: string; paused?: boolean; status?: string | null; error?: string | null; steps?: Array<{ status: string }>; workflowStepResults?: Array<{ status: string }> }): boolean {
|
||||
if (getTaskMergeBlocker(task as any)) return false;
|
||||
return (task.mergeRetries ?? 0) < maxAutoMergeRetries;
|
||||
}
|
||||
|
||||
function enqueueMerge(taskId: string): void {
|
||||
if (mergeActive.has(taskId)) return;
|
||||
mergeActive.add(taskId);
|
||||
mergeQueue.push(taskId);
|
||||
void drainMergeQueue();
|
||||
}
|
||||
|
||||
async function drainMergeQueue(): Promise<void> {
|
||||
if (mergeRunning) return;
|
||||
mergeRunning = true;
|
||||
try {
|
||||
while (mergeQueue.length > 0) {
|
||||
const taskId = mergeQueue.shift()!;
|
||||
try {
|
||||
const settings = await store.getSettings();
|
||||
if (settings.globalPause || settings.enginePaused) {
|
||||
console.log(
|
||||
`[auto-merge] Skipping ${taskId} — ${settings.globalPause ? "global pause" : "engine paused"} active`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!settings.autoMerge) {
|
||||
console.log(`[auto-merge] Skipping ${taskId} — autoMerge disabled`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const task = await store.getTask(taskId);
|
||||
if (!canMergeTask(task as any)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const mergeStrategy = getMergeStrategy(settings);
|
||||
if (mergeStrategy === "pull-request") {
|
||||
console.log(`[auto-merge] Processing PR flow for ${taskId}...`);
|
||||
const result = await processPullRequestMergeTask(store, cwd, taskId, githubClient, getTaskMergeBlocker);
|
||||
if (result === "merged") {
|
||||
console.log(`[auto-merge] ✓ ${taskId} merged via pull request`);
|
||||
} else if (result === "waiting") {
|
||||
console.log(`[auto-merge] … ${taskId} waiting on PR checks or reviews`);
|
||||
}
|
||||
} else {
|
||||
console.log(`[auto-merge] Merging ${taskId}...`);
|
||||
await onMerge(taskId);
|
||||
console.log(`[auto-merge] ✓ ${taskId} merged`);
|
||||
if (task.mergeRetries && task.mergeRetries > 0) {
|
||||
await store.updateTask(taskId, { mergeRetries: 0 });
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
const errorMsg = err.message ?? String(err);
|
||||
console.log(`[auto-merge] ✗ ${taskId}: ${errorMsg}`);
|
||||
|
||||
const settings = await store
|
||||
.getSettings()
|
||||
.catch(() => ({ autoResolveConflicts: true, mergeStrategy: "direct" as const }));
|
||||
const task = await store.getTask(taskId).catch(() => null);
|
||||
const mergeStrategy = getMergeStrategy(settings);
|
||||
|
||||
if (mergeStrategy === "direct") {
|
||||
const isConflictError =
|
||||
errorMsg.includes("conflict") || errorMsg.includes("Conflict");
|
||||
|
||||
if (task && isConflictError) {
|
||||
const currentRetries = task.mergeRetries ?? 0;
|
||||
|
||||
if (settings.autoResolveConflicts !== false && currentRetries < maxAutoMergeRetries) {
|
||||
const newRetryCount = currentRetries + 1;
|
||||
await store.updateTask(taskId, {
|
||||
mergeRetries: newRetryCount,
|
||||
status: null,
|
||||
});
|
||||
|
||||
const delayMs = 5000 * Math.pow(2, currentRetries);
|
||||
console.log(
|
||||
`[auto-merge] ↻ ${taskId}: retry ${newRetryCount}/${maxAutoMergeRetries} in ${delayMs / 1000}s`,
|
||||
);
|
||||
|
||||
setTimeout(() => {
|
||||
enqueueMerge(taskId);
|
||||
}, delayMs);
|
||||
} else {
|
||||
if (currentRetries >= maxAutoMergeRetries) {
|
||||
console.log(
|
||||
`[auto-merge] ⊘ ${taskId}: max retries (${maxAutoMergeRetries}) exceeded — manual resolution required`,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`[auto-merge] ⊘ ${taskId}: autoResolveConflicts disabled — manual resolution required`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
await store.updateTask(taskId, { status: null });
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Non-conflict error - stop auto-retrying until a user intervenes.
|
||||
// This prevents the periodic sweep from re-enqueueing the same
|
||||
// broken merge on every poll cycle.
|
||||
try {
|
||||
await store.updateTask(taskId, {
|
||||
status: null,
|
||||
mergeRetries: maxAutoMergeRetries,
|
||||
error: errorMsg,
|
||||
});
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
await store.updateTask(taskId, {
|
||||
status: null,
|
||||
mergeRetries: maxAutoMergeRetries,
|
||||
error: errorMsg,
|
||||
});
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
mergeActive.delete(taskId);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
mergeRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
store.on("task:moved", async ({ task, to }) => {
|
||||
if (to !== "in-review") return;
|
||||
if (getTaskMergeBlocker(task)) return;
|
||||
try {
|
||||
const settings = await store.getSettings();
|
||||
if (settings.globalPause || settings.enginePaused) return;
|
||||
if (!settings.autoMerge) return;
|
||||
enqueueMerge(task.id);
|
||||
} catch {
|
||||
// ignore settings read errors
|
||||
}
|
||||
});
|
||||
// Get automation store from the engine (initialized by ProjectEngine)
|
||||
const automationStore = engine.getAutomationStore();
|
||||
|
||||
const authStorage = AuthStorage.create();
|
||||
const modelRegistry = new ModelRegistry(authStorage);
|
||||
@@ -735,34 +491,8 @@ export async function runServe(
|
||||
modelRegistry.refresh();
|
||||
}
|
||||
|
||||
const missionAutopilot = new MissionAutopilot(store, store.getMissionStore());
|
||||
|
||||
// ── MissionExecutionLoop: validation cycle orchestration ───────────
|
||||
//
|
||||
// Created alongside MissionAutopilot to handle the validation cycle
|
||||
// (implement → validate → fix → pass).
|
||||
//
|
||||
const missionExecutionLoop = new MissionExecutionLoop({
|
||||
taskStore: store,
|
||||
missionStore: store.getMissionStore(),
|
||||
missionAutopilot: {
|
||||
notifyValidationComplete: async (featureId: string, _status: "passed" | "failed" | "blocked" | "error") => {
|
||||
// Delegate to autopilot after validation completes
|
||||
// Pass the feature's linked taskId to handleTaskCompletion, not the featureId
|
||||
if (missionAutopilot) {
|
||||
const missionStore = store.getMissionStore();
|
||||
const feature = missionStore?.getFeature(featureId);
|
||||
if (feature?.taskId) {
|
||||
await missionAutopilot.handleTaskCompletion(feature.taskId);
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
rootDir: cwd,
|
||||
});
|
||||
|
||||
const app = createServer(store, {
|
||||
onMerge,
|
||||
onMerge: (taskId) => engine.onMerge(taskId),
|
||||
authStorage,
|
||||
modelRegistry,
|
||||
automationStore,
|
||||
@@ -782,320 +512,6 @@ export async function runServe(
|
||||
headless: true,
|
||||
});
|
||||
|
||||
const executorRef: { current: TaskExecutor | null } = { current: null };
|
||||
const triageRef: { current: TriageProcessor | null } = { current: null };
|
||||
|
||||
const selfHealing = new SelfHealingManager(store, {
|
||||
rootDir: cwd,
|
||||
recoverCompletedTask: (task) =>
|
||||
executorRef.current?.recoverCompletedTask(task) ?? Promise.resolve(false),
|
||||
getExecutingTaskIds: () => executorRef.current?.getExecutingTaskIds() ?? new Set(),
|
||||
});
|
||||
const stuckTaskDetector = new StuckTaskDetector(store, {
|
||||
beforeRequeue: (taskId) => selfHealing.checkStuckBudget(taskId),
|
||||
onLoopDetected: (event) =>
|
||||
executorRef.current?.handleLoopDetected(event) ?? Promise.resolve(false),
|
||||
onStuck: (event) => {
|
||||
triageRef.current?.markStuckAborted(event.taskId);
|
||||
executorRef.current?.markStuckAborted(event.taskId, event.shouldRequeue);
|
||||
console.log(
|
||||
`[engine] ⚠ ${event.taskId} stuck (${event.reason}) — ` +
|
||||
`no progress for ${Math.round(event.noProgressMs / 60_000)}min, ` +
|
||||
`${event.activitySinceProgress} events since last progress — ` +
|
||||
`terminated, ${event.shouldRequeue ? "will retry" : "budget exhausted"}`,
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
// ── TriageProcessor: task specification (TASK LANE — receives semaphore) ──
|
||||
//
|
||||
// Receives the task-lane semaphore to ensure specification agents
|
||||
// count toward settings.maxConcurrent alongside execution and merge.
|
||||
//
|
||||
const triage = new TriageProcessor(store, cwd, {
|
||||
semaphore,
|
||||
usageLimitPauser,
|
||||
stuckTaskDetector,
|
||||
agentStore,
|
||||
onSpecifyStart: (t) => console.log(`[engine] Specifying ${t.id}...`),
|
||||
onSpecifyComplete: (t) => console.log(`[engine] ✓ ${t.id} → todo`),
|
||||
onSpecifyError: (t, e) => console.log(`[engine] ✗ ${t.id}: ${e.message}`),
|
||||
});
|
||||
triageRef.current = triage;
|
||||
|
||||
// ── TaskExecutor: task execution (TASK LANE — receives semaphore) ──────────
|
||||
//
|
||||
// Receives the task-lane semaphore to ensure execution agents
|
||||
// count toward settings.maxConcurrent alongside specification and merge.
|
||||
//
|
||||
const executor = new TaskExecutor(store, cwd, {
|
||||
semaphore,
|
||||
pool,
|
||||
usageLimitPauser,
|
||||
stuckTaskDetector,
|
||||
agentStore,
|
||||
onStart: (t, p) => console.log(`[engine] Executing ${t.id} in ${p}`),
|
||||
onComplete: (t) => console.log(`[engine] ✓ ${t.id} → in-review`),
|
||||
onError: (t, e) => console.log(`[engine] ✗ ${t.id}: ${e.message}`),
|
||||
});
|
||||
executorRef.current = executor;
|
||||
|
||||
const settings = await store.getSettings();
|
||||
const prMonitor = new PrMonitor();
|
||||
const prCommentHandler = new PrCommentHandler(store);
|
||||
prMonitor.onNewComments((taskId, prInfo, comments) =>
|
||||
prCommentHandler.handleNewComments(taskId, prInfo, comments),
|
||||
);
|
||||
|
||||
// ── Scheduler: task coordination (TASK LANE — receives semaphore) ──────────
|
||||
//
|
||||
// Receives the task-lane semaphore to ensure task assignment decisions
|
||||
// respect the concurrency limit alongside running execution agents.
|
||||
//
|
||||
const scheduler = new Scheduler(store, {
|
||||
semaphore,
|
||||
prMonitor,
|
||||
missionStore: store.getMissionStore(),
|
||||
missionAutopilot,
|
||||
missionExecutionLoop,
|
||||
onSchedule: (t) => console.log(`[engine] Scheduled ${t.id}`),
|
||||
onBlocked: (t, deps) =>
|
||||
console.log(`[engine] ${t.id} blocked by ${deps.join(", ")}`),
|
||||
onClosedPrFeedback: async (taskId, prInfo, comments) => {
|
||||
await prCommentHandler.createFollowUpTask(taskId, prInfo, comments);
|
||||
},
|
||||
});
|
||||
|
||||
missionAutopilot.setScheduler(scheduler);
|
||||
|
||||
// Post-run callback for memory insight extraction processing
|
||||
const onMemoryInsightRunProcessed = async (
|
||||
schedule: ScheduledTask,
|
||||
result: AutomationRunResult,
|
||||
): Promise<void> => {
|
||||
// Only process the memory insight extraction schedule
|
||||
if (schedule.name !== INSIGHT_EXTRACTION_SCHEDULE_NAME) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Extract the AI step output from the result
|
||||
const stepResults = result.stepResults ?? [];
|
||||
// Step name updated in FN-1477 to include pruning
|
||||
const aiStep = stepResults.find(
|
||||
(sr) => sr.stepName === "Extract Memory Insights and Prune" || sr.stepName === "Extract Memory Insights",
|
||||
);
|
||||
|
||||
if (!aiStep) {
|
||||
console.log(`[memory-audit] No insight extraction step found in ${schedule.name} result`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[memory-audit] Processing memory insight extraction run...`);
|
||||
|
||||
try {
|
||||
const auditReport = await processAndAuditInsightExtraction(cwd, {
|
||||
rawResponse: aiStep.output ?? "",
|
||||
stepSuccess: aiStep.success,
|
||||
runAt: result.startedAt,
|
||||
error: aiStep.error,
|
||||
});
|
||||
|
||||
const pruneStatus = auditReport.pruning.applied
|
||||
? ` | Pruned: ${auditReport.pruning.originalSize} → ${auditReport.pruning.newSize} chars`
|
||||
: ` | Pruning: ${auditReport.pruning.reason}`;
|
||||
|
||||
console.log(
|
||||
`[memory-audit] ✓ Audit complete — Health: ${auditReport.health}, ` +
|
||||
`Insights: ${auditReport.insightsMemory.insightCount}${pruneStatus}`,
|
||||
);
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[memory-audit] ✗ Failed to process insight extraction: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// ── CronRunner: scheduled automation (UTILITY — NO semaphore) ──────────
|
||||
//
|
||||
// ⚠️ UTILITY PATH: CronRunner does NOT receive the task-lane semaphore.
|
||||
//
|
||||
// Uses createAiPromptExecutor (cwd-only factory) for AI execution in
|
||||
// scheduled tasks. By design, automation prompts are independent of
|
||||
// task concurrency limits so they can run regardless of task-lane saturation.
|
||||
//
|
||||
// createAiPromptExecutor takes only `cwd` (no semaphore parameter),
|
||||
// ensuring automation never competes with task-lane agents for slots.
|
||||
//
|
||||
const aiPromptExecutor = await createAiPromptExecutor(cwd);
|
||||
const cronRunner = new CronRunner(store, automationStore, {
|
||||
aiPromptExecutor,
|
||||
onScheduleRunProcessed: onMemoryInsightRunProcessed,
|
||||
});
|
||||
|
||||
// ── Sync insight extraction automation on startup ─────────────────
|
||||
// Run sync BEFORE starting the cron runner to avoid stale config races.
|
||||
// This ensures the insight extraction schedule is created/updated/deleted
|
||||
// before the first tick can execute it.
|
||||
try {
|
||||
await syncInsightExtractionAutomation(automationStore, settings);
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[memory-audit] Failed to sync insight extraction automation: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
|
||||
cronRunner.start();
|
||||
|
||||
triage.start();
|
||||
scheduler.start();
|
||||
missionAutopilot.start();
|
||||
missionExecutionLoop.start();
|
||||
stuckTaskDetector.start();
|
||||
selfHealing.start();
|
||||
|
||||
// ── Startup: recover active missions for validation loop ─────────────
|
||||
// Re-enqueue pending validations from any missions that were interrupted
|
||||
// before the engine was stopped (e.g., features in validating/needs_fix state).
|
||||
void missionExecutionLoop.recoverActiveMissions().catch((err) => {
|
||||
console.error("[engine] Failed to recover active missions:", err);
|
||||
});
|
||||
|
||||
executor.resumeOrphaned().catch((err) =>
|
||||
console.error("[engine] Failed to resume orphaned tasks:", err),
|
||||
);
|
||||
|
||||
if (settings.autoMerge) {
|
||||
const existing = await store.listTasks({ column: "in-review" });
|
||||
const inReview = existing.filter((t) => !getTaskMergeBlocker(t));
|
||||
if (inReview.length > 0) {
|
||||
console.log(
|
||||
`[auto-merge] Startup sweep: enqueueing ${inReview.length} in-review task(s)`,
|
||||
);
|
||||
for (const t of inReview) {
|
||||
enqueueMerge(t.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Always sync semaphore limit on any settings change ────────────
|
||||
// Without this, changing maxConcurrent in the dashboard has no effect
|
||||
// on the semaphore until an unpause transition or merge retry fires.
|
||||
store.on("settings:updated", ({ settings: s }) => {
|
||||
if (s.maxConcurrent !== undefined) {
|
||||
cachedMaxConcurrent = s.maxConcurrent;
|
||||
}
|
||||
});
|
||||
|
||||
store.on("settings:updated", async ({ settings: s, previous: prev }) => {
|
||||
if (prev.globalPause && !s.globalPause) {
|
||||
console.log("[engine] Global unpause — resuming agentic activity");
|
||||
|
||||
executor.resumeOrphaned().catch((err) =>
|
||||
console.error("[engine] Failed to resume orphaned tasks on unpause:", err),
|
||||
);
|
||||
|
||||
if (s.autoMerge) {
|
||||
try {
|
||||
const tasks = await store.listTasks({ column: "in-review" });
|
||||
for (const t of tasks) {
|
||||
if (!getTaskMergeBlocker(t)) {
|
||||
enqueueMerge(t.id);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore errors in unpause sweep
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
store.on("settings:updated", async ({ settings: s, previous: prev }) => {
|
||||
if (prev.enginePaused && !s.enginePaused) {
|
||||
console.log("[engine] Engine unpaused — resuming agentic activity");
|
||||
|
||||
executor.resumeOrphaned().catch((err) =>
|
||||
console.error(
|
||||
"[engine] Failed to resume orphaned tasks on engine unpause:",
|
||||
err,
|
||||
),
|
||||
);
|
||||
|
||||
if (s.autoMerge) {
|
||||
try {
|
||||
const tasks = await store.listTasks({ column: "in-review" });
|
||||
for (const t of tasks) {
|
||||
if (!getTaskMergeBlocker(t)) {
|
||||
enqueueMerge(t.id);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore errors in unpause sweep
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
store.on("settings:updated", async ({ settings: s, previous: prev }) => {
|
||||
if (s.taskStuckTimeoutMs !== prev.taskStuckTimeoutMs) {
|
||||
console.log(
|
||||
`[stuck-detector] Timeout changed to ${s.taskStuckTimeoutMs}ms — running immediate check`,
|
||||
);
|
||||
await stuckTaskDetector.checkNow();
|
||||
}
|
||||
});
|
||||
|
||||
// ── Insight extraction automation sync on settings change ─────────
|
||||
// When insight extraction settings change (enable/disable/schedule/min interval),
|
||||
// resync the automation schedule without requiring a restart.
|
||||
store.on("settings:updated", async ({ settings: s, previous: prev }) => {
|
||||
const insightKeys = [
|
||||
"insightExtractionEnabled",
|
||||
"insightExtractionSchedule",
|
||||
"insightExtractionMinIntervalMs",
|
||||
] as const;
|
||||
|
||||
const relevantKeyChanged = insightKeys.some((key) => s[key] !== prev[key]);
|
||||
if (relevantKeyChanged) {
|
||||
try {
|
||||
await syncInsightExtractionAutomation(automationStore, s);
|
||||
console.log("[memory-audit] Insight extraction automation synced with settings");
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[memory-audit] Failed to sync insight extraction automation: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let shuttingDown = false;
|
||||
let mergeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
async function scheduleMergeRetry(): Promise<void> {
|
||||
if (shuttingDown) return;
|
||||
const currentSettings = await store.getSettings().catch(() => settings);
|
||||
const interval = currentSettings.pollIntervalMs ?? 15_000;
|
||||
mergeRetryTimer = setTimeout(async () => {
|
||||
if (shuttingDown) return;
|
||||
try {
|
||||
const s = await store.getSettings();
|
||||
cachedMaxConcurrent = s.maxConcurrent;
|
||||
if (!s.globalPause && !s.enginePaused && s.autoMerge) {
|
||||
const tasks = await store.listTasks({ column: "in-review" });
|
||||
for (const t of tasks) {
|
||||
if (!getTaskMergeBlocker(t)) {
|
||||
enqueueMerge(t.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore errors in periodic sweep
|
||||
}
|
||||
if (!shuttingDown) {
|
||||
void scheduleMergeRetry();
|
||||
}
|
||||
}, interval);
|
||||
}
|
||||
void scheduleMergeRetry();
|
||||
|
||||
const server = app.listen(selectedPort, selectedHost);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
@@ -1105,17 +521,31 @@ export async function runServe(
|
||||
|
||||
const actualPort = (server.address() as AddressInfo).port;
|
||||
|
||||
let centralCore: CentralCore | null = null;
|
||||
// ── CentralCore: node registration ────────────────────────────────────
|
||||
//
|
||||
// Reuse the shared CentralCore instance created earlier (for ntfyProjectId).
|
||||
// If it wasn't initialized successfully, create a new one for node registration.
|
||||
//
|
||||
let centralCore: CentralCore | null = sharedCentralCore;
|
||||
// sharedCentralCore was already init'd; if null, try again for node registration
|
||||
if (!centralCore) {
|
||||
try {
|
||||
centralCore = new CentralCore();
|
||||
await centralCore.init();
|
||||
} catch {
|
||||
centralCore = null;
|
||||
}
|
||||
}
|
||||
let localNodeId: string | undefined;
|
||||
|
||||
try {
|
||||
centralCore = new CentralCore();
|
||||
await centralCore.init();
|
||||
const nodes = await centralCore.listNodes();
|
||||
const localNode = nodes.find((node) => node.type === "local");
|
||||
if (localNode) {
|
||||
localNodeId = localNode.id;
|
||||
await centralCore.updateNode(localNode.id, { status: "online" });
|
||||
if (centralCore) {
|
||||
const nodes = await centralCore.listNodes();
|
||||
const localNode = nodes.find((node) => node.type === "local");
|
||||
if (localNode) {
|
||||
localNodeId = localNode.id;
|
||||
await centralCore.updateNode(localNode.id, { status: "online" });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
@@ -1133,6 +563,8 @@ export async function runServe(
|
||||
console.log(` Press Ctrl+C to stop`);
|
||||
console.log();
|
||||
|
||||
let shuttingDown = false;
|
||||
|
||||
const shutdown = async () => {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
@@ -1155,23 +587,11 @@ export async function runServe(
|
||||
// Ignore errors getting handle types
|
||||
}
|
||||
|
||||
// Stop heartbeat components first (they reference agentStore)
|
||||
if (triggerScheduler) triggerScheduler.stop();
|
||||
if (heartbeatMonitor) heartbeatMonitor.stop();
|
||||
|
||||
selfHealing.stop();
|
||||
stuckTaskDetector.stop();
|
||||
missionAutopilot.stop();
|
||||
missionExecutionLoop.stop();
|
||||
triage.stop();
|
||||
scheduler.stop();
|
||||
cronRunner.stop();
|
||||
notifier.stop();
|
||||
|
||||
if (mergeRetryTimer) {
|
||||
clearTimeout(mergeRetryTimer);
|
||||
mergeRetryTimer = null;
|
||||
}
|
||||
// Stop the engine (stops all subsystems: runtime, notifier, cronRunner, etc.)
|
||||
await engine.stop().catch((err) => {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[serve] Engine stop error: ${message}`);
|
||||
});
|
||||
|
||||
if (centralCore && localNodeId) {
|
||||
try {
|
||||
|
||||
@@ -36,6 +36,7 @@ export const DEFAULT_PROJECT_SETTINGS = {
|
||||
globalPause: false,
|
||||
enginePaused: false,
|
||||
maxConcurrent: 2,
|
||||
globalMaxConcurrent: 4,
|
||||
maxWorktrees: 4,
|
||||
pollIntervalMs: 15000,
|
||||
groupOverlappingFiles: true,
|
||||
|
||||
@@ -902,6 +902,11 @@ export interface ProjectSettings {
|
||||
/** Maximum number of concurrent AI agents across all activity types
|
||||
* (triage specification, task execution, and merge operations). */
|
||||
maxConcurrent: number;
|
||||
/** System-wide maximum concurrent agents across ALL projects.
|
||||
* When multiple projects are active, the sum of their in-flight agents
|
||||
* will not exceed this limit. Applies to triage, execution, and merge.
|
||||
* Default: 4. When undefined, falls back to CentralCore default (4). */
|
||||
globalMaxConcurrent?: number;
|
||||
maxWorktrees: number;
|
||||
pollIntervalMs: number;
|
||||
groupOverlappingFiles: boolean;
|
||||
|
||||
104
packages/engine/MIGRATION_PLAN.md
Normal file
104
packages/engine/MIGRATION_PLAN.md
Normal file
@@ -0,0 +1,104 @@
|
||||
# ProjectEngine Migration Plan
|
||||
|
||||
Status: In progress
|
||||
Last updated: 2026-04-12
|
||||
|
||||
## Goal
|
||||
|
||||
Consolidate all engine subsystem wiring into `ProjectEngine` so that every code path
|
||||
(single-project CLI, multi-project ProjectManager, child-process worker) gets the full
|
||||
subsystem set from one place. This eliminates the class of bugs where a subsystem is
|
||||
added in one code path but forgotten in another (e.g., TriageProcessor was missing from
|
||||
InProcessRuntime for multi-project mode).
|
||||
|
||||
## Current architecture
|
||||
|
||||
```
|
||||
Single-project CLI (serve.ts / dashboard.ts)
|
||||
└─ Creates subsystems inline + passes to createServer
|
||||
|
||||
Multi-project (ProjectManager)
|
||||
└─ InProcessRuntime / ChildProcessRuntime / RemoteNodeRuntime
|
||||
└─ ChildProcessRuntime → child-process-worker.ts → ProjectEngine
|
||||
```
|
||||
|
||||
## What ProjectEngine now handles
|
||||
|
||||
These subsystems are managed by ProjectEngine and should NOT be duplicated inline:
|
||||
|
||||
| Subsystem | Source |
|
||||
|---|---|
|
||||
| InProcessRuntime (TaskStore, Scheduler, TaskExecutor, TriageProcessor, StuckTaskDetector, AgentSemaphore, WorktreePool, UsageLimitPauser, AgentStore) | via InProcessRuntime |
|
||||
| PrMonitor + PrCommentHandler | ProjectEngine |
|
||||
| NtfyNotifier | ProjectEngine |
|
||||
| CronRunner + AutomationStore | ProjectEngine |
|
||||
| Auto-merge queue (with conflict retry, verification error handling, cooldown retry, buffer failure healing) | ProjectEngine |
|
||||
| Settings event listeners (global pause, unpause, engine unpause, stuck timeout, insight extraction sync) | ProjectEngine |
|
||||
|
||||
## What remains inline in serve.ts / dashboard.ts
|
||||
|
||||
These components are CLI-specific and NOT yet in ProjectEngine. Future migration
|
||||
candidates are marked with priority.
|
||||
|
||||
### High priority (shared across serve.ts and dashboard.ts)
|
||||
|
||||
| Component | Why it's inline | Migration path |
|
||||
|---|---|---|
|
||||
| **MissionAutopilot** | Needs scheduler ref (circular dep via `setScheduler`). Created before engine start. | Add to ProjectEngine. Break circular dep by having ProjectEngine call `setScheduler()` internally after runtime start. |
|
||||
| **MissionExecutionLoop** | Coupled to MissionAutopilot. Needs taskStore + missionStore + rootDir. | Move alongside MissionAutopilot into ProjectEngine. |
|
||||
| **SelfHealingManager** | Needs executor + triage refs for recovery callbacks. Currently uses late-binding `executorRef`/`triageRef`. | Add to ProjectEngine. Wire callbacks to internal runtime's executor/triage. Expose `recoverCompletedTask` etc. |
|
||||
|
||||
### Medium priority (shared but with CLI-specific behavior)
|
||||
|
||||
| Component | Why it's inline | Migration path |
|
||||
|---|---|---|
|
||||
| **HeartbeatMonitor** | Utility path (no semaphore). Needs agentStore, taskStore, rootDir, and CLI-specific callbacks (`onMissed`, `onTerminated` log to console). | Add to ProjectEngine with configurable callbacks. Keep as utility (no semaphore gating). |
|
||||
| **HeartbeatTriggerScheduler** | Paired with HeartbeatMonitor. Needs agentStore + callback to HeartbeatMonitor. | Move alongside HeartbeatMonitor. |
|
||||
| **AuthStorage + ModelRegistry + extension loading** | Pi-coding-agent specific. Discovers extensions, registers providers, syncs OpenRouter models. | Keep in CLI layer — this is auth/model wiring, not engine orchestration. Not a ProjectEngine concern. |
|
||||
|
||||
### Low priority (CLI-specific, keep inline)
|
||||
|
||||
| Component | Reason to keep inline |
|
||||
|---|---|
|
||||
| **PluginStore + PluginLoader** | Plugin system is a dashboard/CLI concern, not engine. |
|
||||
| **`createServer()` call** | HTTP server setup is CLI-specific. |
|
||||
| **Diagnostic utilities** | Process monitoring, memory logging — CLI concern. |
|
||||
| **Port selection** | Interactive prompt — CLI concern. |
|
||||
| **CentralCore node registration** | Registers local node status — serve.ts specific. |
|
||||
| **`onMemoryInsightRunProcessed` callback** | Already passed via `onInsightRunProcessed` to ProjectEngine. The detailed `processAndAuditInsightExtraction` call can stay as the callback impl. |
|
||||
|
||||
## Migration sequence
|
||||
|
||||
### Phase 1 (current — in progress)
|
||||
- [x] Add TriageProcessor to InProcessRuntime
|
||||
- [x] Create ProjectEngine wrapper
|
||||
- [x] Migrate child-process-worker to ProjectEngine
|
||||
- [x] Shared global semaphore from ProjectManager
|
||||
- [x] Move richer merge logic (verification handling, cooldown retry) into ProjectEngine
|
||||
- [ ] Migrate serve.ts to use ProjectEngine (agent in progress)
|
||||
- [ ] Migrate dashboard.ts to use ProjectEngine (agent in progress)
|
||||
|
||||
### Phase 2 (next)
|
||||
- [ ] Move MissionAutopilot + MissionExecutionLoop into ProjectEngine
|
||||
- Add `missionStore` to ProjectEngineOptions
|
||||
- Create MissionAutopilot internally, wire setScheduler after start
|
||||
- Expose via `getMissionAutopilot()` / `getMissionExecutionLoop()`
|
||||
- [ ] Move SelfHealingManager into ProjectEngine
|
||||
- Wire callbacks to internal runtime's executor/triage
|
||||
- No external configuration needed
|
||||
|
||||
### Phase 3 (later)
|
||||
- [ ] Move HeartbeatMonitor + HeartbeatTriggerScheduler into ProjectEngine
|
||||
- Add heartbeat callbacks to ProjectEngineOptions
|
||||
- Keep as utility path (no semaphore)
|
||||
- Expose via `getHeartbeatMonitor()`
|
||||
- [ ] Audit serve.ts / dashboard.ts for remaining inline engine wiring
|
||||
- [ ] Consider making `createServer` accept a ProjectEngine directly
|
||||
|
||||
## Design principles
|
||||
|
||||
1. **ProjectEngine is the single source of truth** for engine subsystem composition
|
||||
2. **CLI layer** only handles: HTTP server, auth, plugins, diagnostics, UI-specific callbacks
|
||||
3. **Callbacks over hardcoding** — ProjectEngine accepts option callbacks for CLI-specific behavior (merge strategy, PR merge, insight processing, etc.)
|
||||
4. **No duplicate subsystems** — if ProjectEngine creates it, CLI must not also create it
|
||||
5. **Dev mode** — dashboard.ts `opts.dev` skips engine start entirely; ProjectEngine handles this via not calling `start()`
|
||||
@@ -3,7 +3,10 @@ import type {
|
||||
Task,
|
||||
CentralCore,
|
||||
Settings,
|
||||
MergeResult,
|
||||
AutomationStore as AutomationStoreType,
|
||||
ScheduledTask,
|
||||
AutomationRunResult,
|
||||
} from "@fusion/core";
|
||||
import { InProcessRuntime } from "./runtimes/in-process-runtime.js";
|
||||
import type { ProjectRuntimeConfig } from "./project-runtime.js";
|
||||
@@ -30,6 +33,12 @@ export interface ProjectEngineOptions {
|
||||
projectId?: string;
|
||||
/** Base URL for ntfy.sh notifications */
|
||||
ntfyBaseUrl?: string;
|
||||
/**
|
||||
* An already-initialized TaskStore to use instead of creating a new one.
|
||||
* When provided, InProcessRuntime will skip TaskStore construction and init().
|
||||
* Useful when the caller (e.g. dashboard.ts) owns and watches the store.
|
||||
*/
|
||||
externalTaskStore?: TaskStore;
|
||||
/**
|
||||
* Returns the merge strategy for the current settings.
|
||||
* If not provided, defaults to "direct".
|
||||
@@ -50,6 +59,11 @@ export interface ProjectEngineOptions {
|
||||
* Invoked after CronRunner completes a memory insight extraction schedule.
|
||||
*/
|
||||
onInsightRunProcessed?: (schedule: unknown, result: unknown) => void | Promise<void>;
|
||||
/**
|
||||
* Whether to skip starting NtfyNotifier. Useful when the caller manages
|
||||
* notifications independently. Defaults to false (notifier is started).
|
||||
*/
|
||||
skipNotifier?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -83,6 +97,8 @@ export class ProjectEngine {
|
||||
private shuttingDown = false;
|
||||
|
||||
private static readonly MAX_AUTO_MERGE_RETRIES = 3;
|
||||
/** 30-minute cooldown before a retry-exhausted task gets another sweep attempt */
|
||||
private static readonly AUTO_MERGE_COOLDOWN_MS = 30 * 60 * 1000;
|
||||
|
||||
// Event handler references for cleanup
|
||||
private settingsHandlers: Array<(...args: any[]) => void> = [];
|
||||
@@ -93,7 +109,11 @@ export class ProjectEngine {
|
||||
centralCore: CentralCore,
|
||||
private options: ProjectEngineOptions = {},
|
||||
) {
|
||||
this.runtime = new InProcessRuntime(config, centralCore);
|
||||
// Pass through externalTaskStore to the runtime config if provided
|
||||
const runtimeConfig: ProjectRuntimeConfig = options.externalTaskStore
|
||||
? { ...config, externalTaskStore: options.externalTaskStore }
|
||||
: config;
|
||||
this.runtime = new InProcessRuntime(runtimeConfig, centralCore);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -113,12 +133,14 @@ export class ProjectEngine {
|
||||
this.prCommentHandler!.handleNewComments(taskId, prInfo, comments),
|
||||
);
|
||||
|
||||
// 3. Initialize NtfyNotifier
|
||||
this.notifier = new NtfyNotifier(store, {
|
||||
projectId: this.options.projectId,
|
||||
ntfyBaseUrl: this.options.ntfyBaseUrl,
|
||||
});
|
||||
await this.notifier.start();
|
||||
// 3. Initialize NtfyNotifier (unless caller manages it externally)
|
||||
if (!this.options.skipNotifier) {
|
||||
this.notifier = new NtfyNotifier(store, {
|
||||
projectId: this.options.projectId,
|
||||
ntfyBaseUrl: this.options.ntfyBaseUrl,
|
||||
});
|
||||
await this.notifier.start();
|
||||
}
|
||||
|
||||
// 4. Initialize AutomationStore + CronRunner
|
||||
try {
|
||||
@@ -129,7 +151,7 @@ export class ProjectEngine {
|
||||
const aiPromptExecutor = await createAiPromptExecutor(cwd);
|
||||
this.cronRunner = new CronRunner(store, this.automationStore, {
|
||||
aiPromptExecutor,
|
||||
onScheduleRunProcessed: this.options.onInsightRunProcessed as any,
|
||||
onScheduleRunProcessed: this.buildInsightRunHandler(cwd),
|
||||
});
|
||||
|
||||
// Sync insight extraction automation on startup
|
||||
@@ -231,15 +253,120 @@ export class ProjectEngine {
|
||||
return this.cronRunner;
|
||||
}
|
||||
|
||||
// ── Auto-merge subsystem ──
|
||||
|
||||
private canMergeTask(task: Task): boolean {
|
||||
const blocker = this.options.getTaskMergeBlocker?.(task);
|
||||
if (blocker) return false;
|
||||
return (task.mergeRetries ?? 0) < ProjectEngine.MAX_AUTO_MERGE_RETRIES;
|
||||
/** Get the AutomationStore (if initialized). */
|
||||
getAutomationStore(): AutomationStoreType | undefined {
|
||||
return this.automationStore;
|
||||
}
|
||||
|
||||
private enqueueMerge(taskId: string): void {
|
||||
/**
|
||||
* Enqueue a task ID for auto-merge if it is not already queued or active.
|
||||
* Exposed publicly so callers can integrate the engine's merge queue with
|
||||
* an external `onMerge` callback (e.g. dashboard's createServer call).
|
||||
*/
|
||||
enqueueMerge(taskId: string): void {
|
||||
this.internalEnqueueMerge(taskId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Directly perform an AI-powered merge for a task (semaphore-gated).
|
||||
* This is the manual "merge now" path, bypassing the auto-merge queue.
|
||||
* Returns the full MergeResult so it can be used as the `onMerge` callback
|
||||
* in createServer().
|
||||
*/
|
||||
async onMerge(taskId: string): Promise<MergeResult> {
|
||||
const store = this.runtime.getTaskStore();
|
||||
const cwd = this.config.workingDirectory;
|
||||
const semaphore = (this.runtime as any).globalSemaphore;
|
||||
const pool = (this.runtime as any).worktreePool;
|
||||
const agentStore = (this.runtime as any).agentStore;
|
||||
const usageLimitPauser = (this.runtime as any).usageLimitPauser;
|
||||
|
||||
const rawMerge = () =>
|
||||
aiMergeTask(store, cwd, taskId, {
|
||||
pool,
|
||||
usageLimitPauser,
|
||||
agentStore,
|
||||
onSession: (session) => {
|
||||
this.activeMergeSession = session;
|
||||
},
|
||||
});
|
||||
|
||||
const result = semaphore
|
||||
? await semaphore.run(rawMerge, PRIORITY_MERGE)
|
||||
: await rawMerge();
|
||||
|
||||
this.activeMergeSession = null;
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Merge eligibility helpers (richer logic from dashboard.ts) ──
|
||||
|
||||
/**
|
||||
* True when a retry-exhausted task in "in-review" has a verification buffer
|
||||
* failure that can be auto-healed by resetting mergeRetries and re-running.
|
||||
*/
|
||||
private hasAutoHealableVerificationBufferFailure(task: {
|
||||
mergeRetries?: number | null;
|
||||
column: string;
|
||||
error?: string | null;
|
||||
log?: Array<{ action?: string }>;
|
||||
}): boolean {
|
||||
if (task.column !== "in-review") return false;
|
||||
if ((task.mergeRetries ?? 0) < ProjectEngine.MAX_AUTO_MERGE_RETRIES) return false;
|
||||
const err = task.error ?? "";
|
||||
const matchesVerificationError =
|
||||
err.includes("Deterministic test verification failed") ||
|
||||
err.includes("Deterministic build verification failed") ||
|
||||
err.includes("Build verification failed") ||
|
||||
err.includes("Test verification failed");
|
||||
if (!matchesVerificationError) return false;
|
||||
|
||||
return (
|
||||
task.log?.some(
|
||||
(entry) =>
|
||||
entry.action?.includes("[verification] test command failed (exit 0)") ||
|
||||
entry.action?.includes("[verification] build command failed (exit 0)") ||
|
||||
entry.action?.includes("output exceeded buffer"),
|
||||
) ?? false
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a retry-exhausted task has been idle long enough for a
|
||||
* 30-minute cooldown merge attempt.
|
||||
*/
|
||||
private isRetryCooldownElapsed(task: { updatedAt?: string | null }): boolean {
|
||||
if (!task.updatedAt) return false;
|
||||
const updated = Date.parse(task.updatedAt);
|
||||
if (Number.isNaN(updated)) return false;
|
||||
return Date.now() - updated >= ProjectEngine.AUTO_MERGE_COOLDOWN_MS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the task is eligible for auto-merge. Uses richer eligibility
|
||||
* checks: merge blocker, retry limit, auto-heal patterns, cooldown elapsed.
|
||||
*/
|
||||
private canMergeTask(task: {
|
||||
id?: string;
|
||||
mergeRetries?: number | null;
|
||||
column: string;
|
||||
paused?: boolean;
|
||||
status?: string | null;
|
||||
error?: string | null;
|
||||
steps?: Array<{ status: string }>;
|
||||
workflowStepResults?: Array<{ status: string }>;
|
||||
log?: Array<{ action?: string }>;
|
||||
updatedAt?: string | null;
|
||||
}): boolean {
|
||||
if (this.options.getTaskMergeBlocker?.(task as Task)) return false;
|
||||
return (
|
||||
(task.mergeRetries ?? 0) < ProjectEngine.MAX_AUTO_MERGE_RETRIES ||
|
||||
this.hasAutoHealableVerificationBufferFailure(task) ||
|
||||
this.isRetryCooldownElapsed(task)
|
||||
);
|
||||
}
|
||||
|
||||
private internalEnqueueMerge(taskId: string): void {
|
||||
if (this.mergeActive.has(taskId)) return;
|
||||
this.mergeActive.add(taskId);
|
||||
this.mergeQueue.push(taskId);
|
||||
@@ -257,23 +384,59 @@ export class ProjectEngine {
|
||||
while (this.mergeQueue.length > 0 && !this.shuttingDown) {
|
||||
const taskId = this.mergeQueue.shift()!;
|
||||
try {
|
||||
// Re-check autoMerge and pause before each merge
|
||||
const settings = await store.getSettings();
|
||||
if (settings.globalPause || settings.enginePaused) {
|
||||
runtimeLog.log(
|
||||
`Auto-merge skipping ${taskId} — ${settings.globalPause ? "global pause" : "engine paused"} active`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!settings.autoMerge) {
|
||||
runtimeLog.log(`Auto-merge skipping ${taskId} — autoMerge disabled`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const task = await store.getTask(taskId);
|
||||
if (!task || task.column !== "in-review") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const settings = await store.getSettings();
|
||||
if (settings.globalPause || settings.enginePaused) break;
|
||||
if (!this.canMergeTask(task as any)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Auto-heal verification buffer failures by resetting retry counter
|
||||
if (this.hasAutoHealableVerificationBufferFailure(task as any)) {
|
||||
await store.logEntry(
|
||||
taskId,
|
||||
"Auto-healing stale deterministic verification buffer failure; retrying merge verification",
|
||||
);
|
||||
await store.updateTask(taskId, { mergeRetries: 0, error: null, status: null });
|
||||
} else if (
|
||||
(task.mergeRetries ?? 0) >= ProjectEngine.MAX_AUTO_MERGE_RETRIES &&
|
||||
this.isRetryCooldownElapsed(task as any)
|
||||
) {
|
||||
await store.logEntry(
|
||||
taskId,
|
||||
`Auto-merge retry cooldown elapsed (${Math.round(ProjectEngine.AUTO_MERGE_COOLDOWN_MS / 60000)}m idle); resetting retries for another attempt`,
|
||||
);
|
||||
await store.updateTask(taskId, { mergeRetries: 0 });
|
||||
}
|
||||
|
||||
const mergeStrategy = this.options.getMergeStrategy?.(settings) ?? "direct";
|
||||
|
||||
if (mergeStrategy === "pull-request" && this.options.processPullRequestMerge) {
|
||||
runtimeLog.log(`Processing PR flow for ${taskId}...`);
|
||||
runtimeLog.log(`Auto-merge processing PR flow for ${taskId}...`);
|
||||
const result = await this.options.processPullRequestMerge(store, cwd, taskId);
|
||||
runtimeLog.log(`PR merge result for ${taskId}: ${result}`);
|
||||
if (result === "merged") {
|
||||
runtimeLog.log(`Auto-merge PR merged: ${taskId}`);
|
||||
} else if (result === "waiting") {
|
||||
runtimeLog.log(`Auto-merge PR waiting: ${taskId}`);
|
||||
}
|
||||
} else {
|
||||
// Direct merge via AI agent, gated by semaphore
|
||||
runtimeLog.log(`Merging ${taskId}...`);
|
||||
runtimeLog.log(`Auto-merge merging ${taskId}...`);
|
||||
const semaphore = (this.runtime as any).globalSemaphore;
|
||||
const pool = (this.runtime as any).worktreePool;
|
||||
const agentStore = (this.runtime as any).agentStore;
|
||||
@@ -296,66 +459,110 @@ export class ProjectEngine {
|
||||
}
|
||||
|
||||
this.activeMergeSession = null;
|
||||
runtimeLog.log(`Merged ${taskId}`);
|
||||
runtimeLog.log(`Auto-merge merged: ${taskId}`);
|
||||
|
||||
// Reset retries on success
|
||||
if (task.mergeRetries && task.mergeRetries > 0) {
|
||||
const latestTask = await store.getTask(taskId).catch(() => null);
|
||||
if (latestTask?.mergeRetries && latestTask.mergeRetries > 0) {
|
||||
await store.updateTask(taskId, { mergeRetries: 0 });
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
this.activeMergeSession = null;
|
||||
const errorMsg = err?.message ?? String(err);
|
||||
runtimeLog.error(`Merge failed for ${taskId}: ${errorMsg}`);
|
||||
runtimeLog.error(`Auto-merge failed for ${taskId}: ${errorMsg}`);
|
||||
|
||||
// Conflict retry with exponential backoff
|
||||
const isConflictError =
|
||||
errorMsg.includes("conflict") || errorMsg.includes("Conflict");
|
||||
const settingsOnErr = await store
|
||||
.getSettings()
|
||||
.catch(() => ({ autoResolveConflicts: true }));
|
||||
const taskOnErr = await store.getTask(taskId).catch(() => null);
|
||||
const mergeStrategyOnErr =
|
||||
this.options.getMergeStrategy?.(settingsOnErr as Settings) ?? "direct";
|
||||
|
||||
if (isConflictError) {
|
||||
// Deterministic verification failure: move back to in-progress
|
||||
const isVerificationError =
|
||||
err?.name === "VerificationError" ||
|
||||
errorMsg.includes("Deterministic test verification failed") ||
|
||||
errorMsg.includes("Deterministic build verification failed");
|
||||
|
||||
if (taskOnErr && isVerificationError) {
|
||||
const failedKind = errorMsg.includes("build verification") ? "build" : "test";
|
||||
try {
|
||||
const task = await store.getTask(taskId);
|
||||
const settings = await store.getSettings();
|
||||
if (
|
||||
task &&
|
||||
settings.autoResolveConflicts !== false &&
|
||||
(task.mergeRetries ?? 0) < ProjectEngine.MAX_AUTO_MERGE_RETRIES
|
||||
) {
|
||||
const retryCount = (task.mergeRetries ?? 0) + 1;
|
||||
await store.updateTask(taskId, {
|
||||
mergeRetries: retryCount,
|
||||
status: null,
|
||||
});
|
||||
|
||||
// Exponential backoff: 5s, 10s, 20s
|
||||
const delayMs = 5000 * Math.pow(2, (task.mergeRetries ?? 0));
|
||||
runtimeLog.log(
|
||||
`Merge conflict retry ${retryCount}/${ProjectEngine.MAX_AUTO_MERGE_RETRIES} for ${taskId} in ${delayMs / 1000}s`,
|
||||
);
|
||||
|
||||
setTimeout(() => {
|
||||
if (!this.shuttingDown) this.enqueueMerge(taskId);
|
||||
}, delayMs);
|
||||
}
|
||||
await store.addTaskComment(
|
||||
taskId,
|
||||
`Deterministic ${failedKind} verification failed during merge. ` +
|
||||
`See the prior [verification] log entry for the truncated command output. ` +
|
||||
`Please fix the failing ${failedKind} and push the update so the merge can retry.`,
|
||||
"agent",
|
||||
);
|
||||
await store.updateTask(taskId, { status: null, mergeRetries: 0, error: null });
|
||||
await store.moveTask(taskId, "in-progress");
|
||||
await store.logEntry(
|
||||
taskId,
|
||||
`Deterministic ${failedKind} verification failed — moved back to in-progress for remediation`,
|
||||
);
|
||||
runtimeLog.log(
|
||||
`Auto-merge: ${taskId} deterministic ${failedKind} verification failed — moved to in-progress`,
|
||||
);
|
||||
} catch {
|
||||
// best-effort retry
|
||||
runtimeLog.error(
|
||||
`Auto-merge: failed to return ${taskId} to in-progress after verification failure`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Verification failure — move back to in-progress
|
||||
const isVerificationError =
|
||||
errorMsg.includes("Verification failed") ||
|
||||
errorMsg.includes("verification failed");
|
||||
if (mergeStrategyOnErr === "direct") {
|
||||
const isConflictError =
|
||||
errorMsg.includes("conflict") || errorMsg.includes("Conflict");
|
||||
|
||||
if (isVerificationError && !isConflictError) {
|
||||
try {
|
||||
const task = await store.getTask(taskId);
|
||||
if (task?.column === "in-review") {
|
||||
await store.moveTask(taskId, "in-progress");
|
||||
runtimeLog.log(`Verification failure — ${taskId} moved back to in-progress`);
|
||||
if (taskOnErr && isConflictError) {
|
||||
const currentRetries = taskOnErr.mergeRetries ?? 0;
|
||||
|
||||
if (
|
||||
(settingsOnErr as Settings).autoResolveConflicts !== false &&
|
||||
currentRetries < ProjectEngine.MAX_AUTO_MERGE_RETRIES
|
||||
) {
|
||||
const newRetryCount = currentRetries + 1;
|
||||
await store.updateTask(taskId, { mergeRetries: newRetryCount, status: null });
|
||||
|
||||
// Exponential backoff: 5s, 10s, 20s
|
||||
const delayMs = 5000 * Math.pow(2, currentRetries);
|
||||
runtimeLog.log(
|
||||
`Auto-merge conflict retry ${newRetryCount}/${ProjectEngine.MAX_AUTO_MERGE_RETRIES} for ${taskId} in ${delayMs / 1000}s`,
|
||||
);
|
||||
setTimeout(() => {
|
||||
if (!this.shuttingDown) this.internalEnqueueMerge(taskId);
|
||||
}, delayMs);
|
||||
} else {
|
||||
// Max retries exceeded or auto-resolve disabled
|
||||
try {
|
||||
await store.updateTask(taskId, { status: null });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Non-conflict error — stop retrying until user intervenes
|
||||
try {
|
||||
await store.updateTask(taskId, {
|
||||
status: null,
|
||||
mergeRetries: ProjectEngine.MAX_AUTO_MERGE_RETRIES,
|
||||
error: errorMsg,
|
||||
});
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
await store.updateTask(taskId, {
|
||||
status: null,
|
||||
mergeRetries: ProjectEngine.MAX_AUTO_MERGE_RETRIES,
|
||||
error: errorMsg,
|
||||
});
|
||||
} catch {
|
||||
// best-effort
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
@@ -375,7 +582,7 @@ export class ProjectEngine {
|
||||
const settings = await store.getSettings();
|
||||
if (settings.globalPause || settings.enginePaused) return;
|
||||
if (!settings.autoMerge) return;
|
||||
this.enqueueMerge(task.id);
|
||||
this.internalEnqueueMerge(task.id);
|
||||
} catch {
|
||||
// ignore settings read errors
|
||||
}
|
||||
@@ -389,11 +596,11 @@ export class ProjectEngine {
|
||||
if (!settings.autoMerge) return;
|
||||
|
||||
const tasks = await store.listTasks({ column: "in-review" });
|
||||
const eligible = tasks.filter((t) => this.canMergeTask(t));
|
||||
const eligible = tasks.filter((t) => this.canMergeTask(t as any));
|
||||
if (eligible.length > 0) {
|
||||
runtimeLog.log(`Auto-merge startup sweep: enqueueing ${eligible.length} task(s)`);
|
||||
for (const t of eligible) {
|
||||
this.enqueueMerge(t.id);
|
||||
this.internalEnqueueMerge(t.id);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
@@ -412,8 +619,8 @@ export class ProjectEngine {
|
||||
if (!settings.globalPause && !settings.enginePaused && settings.autoMerge) {
|
||||
const tasks = await store.listTasks({ column: "in-review" });
|
||||
for (const t of tasks) {
|
||||
if (this.canMergeTask(t)) {
|
||||
this.enqueueMerge(t.id);
|
||||
if (this.canMergeTask(t as any)) {
|
||||
this.internalEnqueueMerge(t.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -451,7 +658,13 @@ export class ProjectEngine {
|
||||
this.settingsHandlers.push(onGlobalPause);
|
||||
|
||||
// 2. Global unpause — resume orphaned tasks + sweep in-review
|
||||
const onGlobalUnpause = async ({ settings: s, previous: prev }: { settings: Settings; previous: Settings }) => {
|
||||
const onGlobalUnpause = async ({
|
||||
settings: s,
|
||||
previous: prev,
|
||||
}: {
|
||||
settings: Settings;
|
||||
previous: Settings;
|
||||
}) => {
|
||||
if (prev.globalPause && !s.globalPause) {
|
||||
runtimeLog.log("Global unpause — resuming agentic activity");
|
||||
|
||||
@@ -460,17 +673,21 @@ export class ProjectEngine {
|
||||
executor?.resumeOrphaned?.().catch((err: Error) =>
|
||||
runtimeLog.error("Failed to resume orphaned tasks on unpause:", err),
|
||||
);
|
||||
} catch { /* ignore */ }
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
if (s.autoMerge) {
|
||||
try {
|
||||
const tasks = await store.listTasks({ column: "in-review" });
|
||||
for (const t of tasks) {
|
||||
if (this.canMergeTask(t)) {
|
||||
this.enqueueMerge(t.id);
|
||||
if (this.canMergeTask(t as any)) {
|
||||
this.internalEnqueueMerge(t.id);
|
||||
}
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -478,7 +695,13 @@ export class ProjectEngine {
|
||||
this.settingsHandlers.push(onGlobalUnpause);
|
||||
|
||||
// 3. Engine unpause — same as global unpause
|
||||
const onEngineUnpause = async ({ settings: s, previous: prev }: { settings: Settings; previous: Settings }) => {
|
||||
const onEngineUnpause = async ({
|
||||
settings: s,
|
||||
previous: prev,
|
||||
}: {
|
||||
settings: Settings;
|
||||
previous: Settings;
|
||||
}) => {
|
||||
if (prev.enginePaused && !s.enginePaused) {
|
||||
runtimeLog.log("Engine unpaused — resuming agentic activity");
|
||||
|
||||
@@ -487,17 +710,21 @@ export class ProjectEngine {
|
||||
executor?.resumeOrphaned?.().catch((err: Error) =>
|
||||
runtimeLog.error("Failed to resume orphaned tasks on engine unpause:", err),
|
||||
);
|
||||
} catch { /* ignore */ }
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
if (s.autoMerge) {
|
||||
try {
|
||||
const tasks = await store.listTasks({ column: "in-review" });
|
||||
for (const t of tasks) {
|
||||
if (this.canMergeTask(t)) {
|
||||
this.enqueueMerge(t.id);
|
||||
if (this.canMergeTask(t as any)) {
|
||||
this.internalEnqueueMerge(t.id);
|
||||
}
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -505,7 +732,13 @@ export class ProjectEngine {
|
||||
this.settingsHandlers.push(onEngineUnpause);
|
||||
|
||||
// 4. Stuck task timeout change — trigger immediate check
|
||||
const onStuckTimeoutChange = async ({ settings: s, previous: prev }: { settings: Settings; previous: Settings }) => {
|
||||
const onStuckTimeoutChange = async ({
|
||||
settings: s,
|
||||
previous: prev,
|
||||
}: {
|
||||
settings: Settings;
|
||||
previous: Settings;
|
||||
}) => {
|
||||
if (s.taskStuckTimeoutMs !== prev.taskStuckTimeoutMs) {
|
||||
runtimeLog.log(
|
||||
`Stuck task timeout changed to ${s.taskStuckTimeoutMs}ms — running immediate check`,
|
||||
@@ -513,23 +746,29 @@ export class ProjectEngine {
|
||||
try {
|
||||
const detector = (this.runtime as any).stuckTaskDetector;
|
||||
await detector?.checkNow?.();
|
||||
} catch { /* ignore */ }
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
};
|
||||
store.on("settings:updated", onStuckTimeoutChange);
|
||||
this.settingsHandlers.push(onStuckTimeoutChange);
|
||||
|
||||
// 5. Insight extraction settings change — sync automation
|
||||
const onInsightSettingsChange = async ({ settings: s, previous: prev }: { settings: Settings; previous: Settings }) => {
|
||||
const onInsightSettingsChange = async ({
|
||||
settings: s,
|
||||
previous: prev,
|
||||
}: {
|
||||
settings: Settings;
|
||||
previous: Settings;
|
||||
}) => {
|
||||
const insightKeys = [
|
||||
"insightExtractionEnabled",
|
||||
"insightExtractionSchedule",
|
||||
"insightExtractionMinIntervalMs",
|
||||
] as const;
|
||||
|
||||
const changed = insightKeys.some(
|
||||
(key) => (s as any)[key] !== (prev as any)[key],
|
||||
);
|
||||
const changed = insightKeys.some((key) => (s as any)[key] !== (prev as any)[key]);
|
||||
if (!changed || !this.automationStore) return;
|
||||
|
||||
try {
|
||||
@@ -548,4 +787,81 @@ export class ProjectEngine {
|
||||
store.on("settings:updated", onInsightSettingsChange);
|
||||
this.settingsHandlers.push(onInsightSettingsChange);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the onScheduleRunProcessed callback for CronRunner.
|
||||
* Chains the built-in processAndAuditInsightExtraction with any
|
||||
* caller-provided onInsightRunProcessed callback.
|
||||
*/
|
||||
private buildInsightRunHandler(
|
||||
cwd: string,
|
||||
): (schedule: ScheduledTask, result: AutomationRunResult) => Promise<void> {
|
||||
const callerCallback = this.options.onInsightRunProcessed;
|
||||
|
||||
return async (schedule: ScheduledTask, result: AutomationRunResult): Promise<void> => {
|
||||
// Invoke caller-provided callback first (e.g. for test hooks)
|
||||
if (callerCallback) {
|
||||
try {
|
||||
await callerCallback(schedule, result);
|
||||
} catch (err) {
|
||||
runtimeLog.warn(
|
||||
"onInsightRunProcessed callback error:",
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Run built-in processAndAuditInsightExtraction
|
||||
try {
|
||||
const { INSIGHT_EXTRACTION_SCHEDULE_NAME, processAndAuditInsightExtraction } =
|
||||
await import("@fusion/core");
|
||||
|
||||
if (
|
||||
typeof INSIGHT_EXTRACTION_SCHEDULE_NAME !== "string" ||
|
||||
typeof processAndAuditInsightExtraction !== "function"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (schedule.name !== INSIGHT_EXTRACTION_SCHEDULE_NAME) {
|
||||
return;
|
||||
}
|
||||
|
||||
const stepResults = result.stepResults ?? [];
|
||||
const aiStep = stepResults.find(
|
||||
(sr) =>
|
||||
sr.stepName === "Extract Memory Insights and Prune" ||
|
||||
sr.stepName === "Extract Memory Insights",
|
||||
);
|
||||
|
||||
if (!aiStep) {
|
||||
runtimeLog.log(`No insight extraction step found in ${schedule.name} result`);
|
||||
return;
|
||||
}
|
||||
|
||||
runtimeLog.log("Processing memory insight extraction run...");
|
||||
|
||||
const auditReport = await processAndAuditInsightExtraction(cwd, {
|
||||
rawResponse: aiStep.output ?? "",
|
||||
stepSuccess: aiStep.success,
|
||||
runAt: result.startedAt,
|
||||
error: aiStep.error,
|
||||
});
|
||||
|
||||
const pruneStatus = auditReport.pruning.applied
|
||||
? ` | Pruned: ${auditReport.pruning.originalSize} -> ${auditReport.pruning.newSize} chars`
|
||||
: ` | Pruning: ${auditReport.pruning.reason}`;
|
||||
|
||||
runtimeLog.log(
|
||||
`Memory audit complete — Health: ${auditReport.health}, ` +
|
||||
`Insights: ${auditReport.insightsMemory.insightCount}${pruneStatus}`,
|
||||
);
|
||||
} catch (err) {
|
||||
runtimeLog.warn(
|
||||
"Failed to process insight extraction:",
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,6 +95,8 @@ export class ProjectManager extends EventEmitter<ProjectManagerEvents> {
|
||||
private runtimes = new Map<string, ProjectRuntime>();
|
||||
private projectNames = new Map<string, string>();
|
||||
private globalSemaphore: AgentSemaphore;
|
||||
/** Mutable limit read by the shared semaphore's getter function. */
|
||||
private currentGlobalLimit = 4;
|
||||
|
||||
/**
|
||||
* @param centralCore - CentralCore reference for global coordination
|
||||
@@ -103,11 +105,10 @@ export class ProjectManager extends EventEmitter<ProjectManagerEvents> {
|
||||
super();
|
||||
this.setMaxListeners(100);
|
||||
|
||||
// Initialize global semaphore with limit from CentralCore
|
||||
this.globalSemaphore = new AgentSemaphore(() => {
|
||||
// This will be updated dynamically from CentralCore
|
||||
return 4; // Default, will refresh
|
||||
});
|
||||
// Initialize global semaphore with a getter that reads the mutable limit.
|
||||
// This single semaphore instance is shared across all runtimes so
|
||||
// cross-project concurrency is enforced correctly.
|
||||
this.globalSemaphore = new AgentSemaphore(() => this.currentGlobalLimit);
|
||||
|
||||
// Refresh the global limit periodically
|
||||
this.refreshGlobalLimit();
|
||||
@@ -122,9 +123,7 @@ export class ProjectManager extends EventEmitter<ProjectManagerEvents> {
|
||||
private async refreshGlobalLimit(): Promise<void> {
|
||||
try {
|
||||
const state = await this.centralCore.getGlobalConcurrencyState();
|
||||
// Update semaphore limit dynamically
|
||||
// Note: AgentSemaphore reads limit via getter, so we update the source
|
||||
this.globalSemaphore = new AgentSemaphore(() => state.globalMaxConcurrent);
|
||||
this.currentGlobalLimit = state.globalMaxConcurrent;
|
||||
} catch (error) {
|
||||
projectManagerLog.warn("Failed to refresh global concurrency limit:", error);
|
||||
}
|
||||
@@ -161,8 +160,11 @@ export class ProjectManager extends EventEmitter<ProjectManagerEvents> {
|
||||
|
||||
// Create appropriate runtime based on isolation mode
|
||||
let runtime: ProjectRuntime;
|
||||
// Inject the shared global semaphore so all runtimes share one concurrency pool.
|
||||
const configWithSemaphore = { ...config, globalSemaphore: this.globalSemaphore };
|
||||
|
||||
if (config.isolationMode === "child-process") {
|
||||
runtime = new ChildProcessRuntime(config, this.centralCore);
|
||||
runtime = new ChildProcessRuntime(configWithSemaphore, this.centralCore);
|
||||
} else {
|
||||
let assignedNode = undefined;
|
||||
|
||||
@@ -184,7 +186,7 @@ export class ProjectManager extends EventEmitter<ProjectManagerEvents> {
|
||||
});
|
||||
} else {
|
||||
// Default to local in-process runtime (includes unassigned + local-node assigned)
|
||||
runtime = new InProcessRuntime(config, this.centralCore);
|
||||
runtime = new InProcessRuntime(configWithSemaphore, this.centralCore);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -45,6 +45,16 @@ export interface ProjectRuntimeConfig {
|
||||
maxWorktrees: number;
|
||||
/** Optional project settings override */
|
||||
settings?: ProjectSettings;
|
||||
/** Shared global semaphore from ProjectManager. When provided, the runtime
|
||||
* uses this semaphore for concurrency control instead of creating its own.
|
||||
* This ensures cross-project concurrency limits are enforced. */
|
||||
globalSemaphore?: import("./concurrency.js").AgentSemaphore;
|
||||
/**
|
||||
* An already-initialized TaskStore to use instead of creating a new one.
|
||||
* When provided, the runtime will skip TaskStore construction and init().
|
||||
* Useful when the caller (e.g. dashboard.ts) owns and watches the store.
|
||||
*/
|
||||
externalTaskStore?: TaskStore;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -13,13 +13,15 @@ const mockState = vi.hoisted(() => ({
|
||||
runtimes: [] as any[],
|
||||
}));
|
||||
|
||||
vi.mock("../logger.js", () => ({
|
||||
runtimeLog: {
|
||||
log: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
},
|
||||
}));
|
||||
vi.mock("../logger.js", () => {
|
||||
const mockLogger = { log: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
||||
return {
|
||||
runtimeLog: mockLogger,
|
||||
createLogger: () => mockLogger,
|
||||
schedulerLog: mockLogger,
|
||||
triageLog: mockLogger,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@fusion/core", () => ({
|
||||
CentralCore: class MockCentralCore {},
|
||||
@@ -88,6 +90,21 @@ vi.mock("./in-process-runtime.js", () => {
|
||||
return { InProcessRuntime: MockInProcessRuntime };
|
||||
});
|
||||
|
||||
vi.mock("../project-engine.js", async () => {
|
||||
const { InProcessRuntime } = await import("./in-process-runtime.js");
|
||||
class MockProjectEngine {
|
||||
private runtime: any;
|
||||
constructor(config: any, centralCore: any, _options?: any) {
|
||||
this.runtime = new InProcessRuntime(config, centralCore);
|
||||
}
|
||||
start = vi.fn(async () => { await this.runtime.start(); });
|
||||
stop = vi.fn(async () => { await this.runtime.stop(); });
|
||||
getRuntime = vi.fn(() => this.runtime);
|
||||
getTaskStore = vi.fn(() => null);
|
||||
}
|
||||
return { ProjectEngine: MockProjectEngine };
|
||||
});
|
||||
|
||||
type MockWorker = {
|
||||
handlers: Map<string, (payload: unknown) => Promise<unknown> | unknown>;
|
||||
onCommand: ReturnType<typeof vi.fn>;
|
||||
@@ -348,7 +365,9 @@ describe("child-process-worker", () => {
|
||||
expect(runtime.stop).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
expect(worker.shutdown).toHaveBeenCalledTimes(1);
|
||||
await vi.waitFor(() => {
|
||||
expect(worker.shutdown).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
it("SIGINT stops runtime and shuts down IPC worker", async () => {
|
||||
@@ -363,6 +382,8 @@ describe("child-process-worker", () => {
|
||||
expect(runtime.stop).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
expect(worker.shutdown).toHaveBeenCalledTimes(1);
|
||||
await vi.waitFor(() => {
|
||||
expect(worker.shutdown).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -89,6 +89,7 @@ export class InProcessRuntime
|
||||
private routineRunner?: RoutineRunner;
|
||||
private routineScheduler?: RoutineScheduler;
|
||||
private missionExecutionLoop?: MissionExecutionLoop;
|
||||
private missionAutopilot?: MissionAutopilot;
|
||||
private triageProcessor?: TriageProcessor;
|
||||
|
||||
/**
|
||||
@@ -124,11 +125,16 @@ export class InProcessRuntime
|
||||
runtimeLog.log(`Starting InProcessRuntime for project ${this.config.projectId}`);
|
||||
|
||||
try {
|
||||
// 1. Initialize TaskStore
|
||||
// 1. Initialize TaskStore (use external if provided, otherwise create new)
|
||||
const { TaskStore, PluginStore: PluginStoreClass, PluginLoader: PluginLoaderClass } = await import("@fusion/core");
|
||||
this.taskStore = new TaskStore(this.config.workingDirectory);
|
||||
await this.taskStore.init();
|
||||
runtimeLog.log(`TaskStore initialized for project ${this.config.projectId}`);
|
||||
if (this.config.externalTaskStore) {
|
||||
this.taskStore = this.config.externalTaskStore;
|
||||
runtimeLog.log(`TaskStore provided externally for project ${this.config.projectId}`);
|
||||
} else {
|
||||
this.taskStore = new TaskStore(this.config.workingDirectory);
|
||||
await this.taskStore.init();
|
||||
runtimeLog.log(`TaskStore initialized for project ${this.config.projectId}`);
|
||||
}
|
||||
|
||||
// 2. Initialize Plugin system (PluginStore + PluginLoader + PluginRunner)
|
||||
this.pluginStore = new PluginStoreClass(this.taskStore.getFusionDir());
|
||||
@@ -164,15 +170,21 @@ export class InProcessRuntime
|
||||
);
|
||||
}
|
||||
|
||||
// 4. Initialize global semaphore from CentralCore
|
||||
const globalLimit = await this.getGlobalConcurrencyLimit();
|
||||
this.globalSemaphore = new AgentSemaphore(() => globalLimit);
|
||||
// 4. Initialize global semaphore — use shared one from ProjectManager if provided,
|
||||
// otherwise create a local one from CentralCore (single-project mode).
|
||||
if (this.config.globalSemaphore) {
|
||||
this.globalSemaphore = this.config.globalSemaphore;
|
||||
} else {
|
||||
const globalLimit = await this.getGlobalConcurrencyLimit();
|
||||
this.globalSemaphore = new AgentSemaphore(() => globalLimit);
|
||||
}
|
||||
|
||||
// 5. Initialize Scheduler
|
||||
const missionStore = this.taskStore.getMissionStore();
|
||||
const missionAutopilot = missionStore
|
||||
this.missionAutopilot = missionStore
|
||||
? new MissionAutopilot(this.taskStore, missionStore)
|
||||
: undefined;
|
||||
const missionAutopilot = this.missionAutopilot;
|
||||
|
||||
// Initialize MissionExecutionLoop for validation cycle handling
|
||||
const missionExecutionLoop = missionStore
|
||||
@@ -482,6 +494,9 @@ export class InProcessRuntime
|
||||
void this.scheduler.reconcileAllMissionFeatures();
|
||||
}
|
||||
|
||||
// 14. Start MissionAutopilot background polling
|
||||
this.missionAutopilot?.start();
|
||||
|
||||
this.setStatus("active");
|
||||
runtimeLog.log(`InProcessRuntime started for project ${this.config.projectId}`);
|
||||
} catch (error) {
|
||||
@@ -563,6 +578,12 @@ export class InProcessRuntime
|
||||
runtimeLog.log("Scheduler stopped");
|
||||
}
|
||||
|
||||
// 7. Stop mission autopilot background polling
|
||||
if (this.missionAutopilot) {
|
||||
this.missionAutopilot.stop();
|
||||
runtimeLog.log("MissionAutopilot stopped");
|
||||
}
|
||||
|
||||
// 7. Stop mission execution loop
|
||||
if (this.missionExecutionLoop) {
|
||||
this.missionExecutionLoop.stop();
|
||||
@@ -709,6 +730,22 @@ export class InProcessRuntime
|
||||
return this.triageProcessor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the MissionAutopilot instance (if initialized).
|
||||
* Returns undefined when no MissionStore is available.
|
||||
*/
|
||||
getMissionAutopilot(): MissionAutopilot | undefined {
|
||||
return this.missionAutopilot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the MissionExecutionLoop instance (if initialized).
|
||||
* Returns undefined when no MissionStore is available.
|
||||
*/
|
||||
getMissionExecutionLoop(): MissionExecutionLoop | undefined {
|
||||
return this.missionExecutionLoop;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a heartbeat run for an agent.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user