1231db4a1d
This merge introduces a memory file markdown preview feature (FN-3584) with corresponding documentation, refines the AgentDetailView and AgentLogViewer components in the dashboard, and adds defensive collision handling for worktree operations during manual task moves (FN-3583). Fusion-Task-Id: FN-3584
2244 lines
87 KiB
TypeScript
2244 lines
87 KiB
TypeScript
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||
import type {
|
||
TaskStore,
|
||
Task,
|
||
CentralCore,
|
||
Settings,
|
||
MergeResult,
|
||
AutomationStore as AutomationStoreType,
|
||
ScheduledTask,
|
||
AutomationRunResult,
|
||
} from "@fusion/core";
|
||
import { compareTasksByPriorityThenAgeAndId, sortTasksByPriorityThenAgeAndId } from "@fusion/core";
|
||
import { execFile } from "node:child_process";
|
||
import { promisify } from "node:util";
|
||
import { InProcessRuntime } from "./runtimes/in-process-runtime.js";
|
||
import type { ProjectRuntimeConfig } from "./project-runtime.js";
|
||
import { PrMonitor } from "./pr-monitor.js";
|
||
import { PrCommentHandler } from "./pr-comment-handler.js";
|
||
import { NtfyNotifier } from "./notifier.js";
|
||
import { NotificationService } from "./notification/index.js";
|
||
import { GridlockDetector } from "./gridlock-detector.js";
|
||
import { CronRunner, createAiPromptExecutor } from "./cron-runner.js";
|
||
import type { RoutineRunner } from "./routine-runner.js";
|
||
import { aiMergeTask } from "./merger.js";
|
||
import { PRIORITY_MERGE } from "./concurrency.js";
|
||
import { runtimeLog } from "./logger.js";
|
||
import type { HeartbeatTriggerScheduler } from "./agent-heartbeat.js";
|
||
import { ResearchOrchestrator } from "./research-orchestrator.js";
|
||
import { ResearchStepRunner } from "./research-step-runner.js";
|
||
import { TunnelProcessManager } from "./remote-access/tunnel-process-manager.js";
|
||
import type {
|
||
ExternalTunnelInfo,
|
||
TunnelProvider,
|
||
TunnelProviderConfig,
|
||
TunnelRestoreDiagnostics,
|
||
TunnelRestoreReasonCode,
|
||
TunnelStatusSnapshot,
|
||
} from "./remote-access/types.js";
|
||
|
||
/**
|
||
* Callback for processing pull-request merge strategy.
|
||
* Injected from the CLI layer since it depends on GitHubClient.
|
||
*/
|
||
export type ProcessPullRequestMergeFn = (
|
||
store: TaskStore,
|
||
cwd: string,
|
||
taskId: string,
|
||
) => Promise<"merged" | "waiting" | "skipped">;
|
||
|
||
const execFileAsync = promisify(execFile);
|
||
|
||
/**
|
||
* Delay between a task moving to in-review and auto-merge being enqueued.
|
||
* Gives the executor's finally block time to complete session disposal,
|
||
* child-agent termination, and any in-flight reviewer teardown so the merger
|
||
* doesn't start emitting logs while the executor is still cleaning up. See
|
||
* FN-2910 for the observed overlap symptom.
|
||
*/
|
||
const MERGE_HANDOFF_GRACE_MS = 300;
|
||
|
||
interface RemoteLifecycleEvaluation {
|
||
provider: TunnelProvider;
|
||
config?: TunnelProviderConfig;
|
||
reason?: TunnelRestoreReasonCode;
|
||
message?: string;
|
||
}
|
||
|
||
const isRemoteActive = (ra: Settings["remoteAccess"] | undefined): boolean =>
|
||
ra?.activeProvider != null && (ra.providers[ra.activeProvider]?.enabled ?? false);
|
||
|
||
function formatErrorDetails(error: unknown): { message: string; detail: string } {
|
||
if (error instanceof Error) {
|
||
return {
|
||
message: error.message || error.name,
|
||
detail: error.stack ?? `${error.name}: ${error.message}`,
|
||
};
|
||
}
|
||
const detail = String(error);
|
||
return { message: detail, detail };
|
||
}
|
||
|
||
export interface AutomationSubsystemHealth {
|
||
status: "not-initialized" | "initializing" | "ready" | "degraded";
|
||
message: string;
|
||
updatedAt: string;
|
||
}
|
||
|
||
export interface ProjectEngineOptions {
|
||
/** Project identifier for notification deep links */
|
||
projectId?: string;
|
||
/** Base URL for ntfy.sh notifications */
|
||
ntfyBaseUrl?: string;
|
||
/**
|
||
* An already-initialized TaskStore to use instead of creating a new one.
|
||
* When provided, InProcessRuntime will skip TaskStore construction and init().
|
||
* Useful when the caller (e.g. dashboard.ts) owns and watches the store.
|
||
*/
|
||
externalTaskStore?: TaskStore;
|
||
/**
|
||
* Returns the merge strategy for the current settings.
|
||
* If not provided, defaults to "direct".
|
||
*/
|
||
getMergeStrategy?: (settings: Settings) => "direct" | "pull-request";
|
||
/**
|
||
* Processes a pull-request merge flow. Required when merge strategy
|
||
* can be "pull-request". Injected from CLI layer.
|
||
*/
|
||
processPullRequestMerge?: ProcessPullRequestMergeFn;
|
||
/**
|
||
* Returns the merge blocker reason for a task, or null/undefined if
|
||
* the task is eligible for merge. Imported from @fusion/core.
|
||
*/
|
||
getTaskMergeBlocker?: (task: Task) => string | null | undefined;
|
||
/**
|
||
* Callback for insight extraction run processing.
|
||
* Invoked after CronRunner completes a memory insight extraction schedule.
|
||
*/
|
||
onInsightRunProcessed?: (schedule: unknown, result: unknown) => void | Promise<void>;
|
||
/**
|
||
* Whether to skip starting NtfyNotifier. Useful when the caller manages
|
||
* notifications independently. Defaults to false (notifier is started).
|
||
*/
|
||
skipNotifier?: boolean;
|
||
}
|
||
|
||
/**
|
||
* ProjectEngine composes an InProcessRuntime with the higher-level
|
||
* subsystems that were previously wired inline in serve.ts / dashboard.ts:
|
||
*
|
||
* - **Auto-merge queue** — serialized merge with conflict retry, semaphore gating
|
||
* - **PrMonitor + PrCommentHandler** — GitHub PR feedback loop
|
||
* - **NotificationService** — provider-driven push notifications
|
||
* - **CronRunner + AutomationStore** — scheduled automations
|
||
* - **Settings event listeners** — dynamic reconfiguration
|
||
*
|
||
* This ensures every InProcessRuntime (single-project CLI or multi-project
|
||
* via ProjectManager) gets the full subsystem set, eliminating the class of
|
||
* bugs where a subsystem is forgotten in one code path.
|
||
*/
|
||
export class ProjectEngine {
|
||
private runtime: InProcessRuntime;
|
||
private started = false;
|
||
private prMonitor?: PrMonitor;
|
||
private prCommentHandler?: PrCommentHandler;
|
||
private notifier?: NtfyNotifier;
|
||
private notificationService?: NotificationService;
|
||
private gridlockDetector?: GridlockDetector;
|
||
private cronRunner?: CronRunner;
|
||
private automationStore?: AutomationStoreType;
|
||
private researchOrchestrator?: ResearchOrchestrator;
|
||
private remoteTunnelManager?: TunnelProcessManager;
|
||
private remoteTunnelRestoreDiagnostics: TunnelRestoreDiagnostics = {
|
||
outcome: "skipped",
|
||
reason: "not_attempted",
|
||
at: new Date().toISOString(),
|
||
provider: null,
|
||
};
|
||
private automationSubsystemHealth: AutomationSubsystemHealth = {
|
||
status: "not-initialized",
|
||
message: "Automation subsystem has not been initialized",
|
||
updatedAt: new Date().toISOString(),
|
||
};
|
||
|
||
// ── Auto-merge state ──
|
||
private mergeQueue: string[] = [];
|
||
private mergeActive = new Set<string>();
|
||
private pausedReviewTaskIds = new Set<string>();
|
||
private mergeRunning = false;
|
||
private activeMergeSession: { dispose: () => void } | null = null;
|
||
private activeMergeTaskId: string | null = null;
|
||
private mergeAbortController: AbortController | null = null;
|
||
private mergeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
||
|
||
/**
|
||
* Pending manual merge resolvers — keyed by taskId.
|
||
* When `onMerge` is called, the task is enqueued like auto-merge but a
|
||
* Promise is stored here so the caller can await the result.
|
||
*/
|
||
private manualMergeResolvers = new Map<
|
||
string,
|
||
{ resolve: (result: MergeResult) => void; reject: (err: Error) => void }
|
||
>();
|
||
private shuttingDown = false;
|
||
|
||
private static readonly MAX_AUTO_MERGE_RETRIES = 3;
|
||
/** Cap on outer in-review→in-progress bounces caused by deterministic
|
||
* verification failures during auto-merge. After this many failed merges
|
||
* for the same task, we stop bouncing it back, mark it failed, and create
|
||
* a follow-up triage task so a fresh agent (or human) can investigate
|
||
* the underlying flake/regression instead of looping forever. */
|
||
private static readonly MAX_VERIFICATION_FAILURE_BOUNCES = 3;
|
||
/** Cap on outer in-review→in-progress bounces caused by auto-merge conflict
|
||
* retries being exhausted. After this many bounces the task is parked in
|
||
* in-review with status=failed and a follow-up task is created, so the
|
||
* 30-minute cooldown sweep cannot loop forever on a merge that requires
|
||
* human intervention. */
|
||
private static readonly MAX_MERGE_CONFLICT_BOUNCES = 2;
|
||
/** 30-minute cooldown before a retry-exhausted task gets another sweep attempt */
|
||
private static readonly AUTO_MERGE_COOLDOWN_MS = 30 * 60 * 1000;
|
||
|
||
// Event handler references for cleanup
|
||
private settingsHandlers: Array<(...args: any[]) => void> = [];
|
||
private taskMovedHandler?: (...args: any[]) => void;
|
||
private taskUpdatedHandler?: (...args: any[]) => void;
|
||
|
||
constructor(
|
||
private config: ProjectRuntimeConfig,
|
||
centralCore: CentralCore,
|
||
private options: ProjectEngineOptions = {},
|
||
) {
|
||
// Pass through externalTaskStore to the runtime config if provided
|
||
const runtimeConfig: ProjectRuntimeConfig = options.externalTaskStore
|
||
? { ...config, externalTaskStore: options.externalTaskStore }
|
||
: config;
|
||
this.runtime = new InProcessRuntime(runtimeConfig, centralCore);
|
||
// Let the runtime's SelfHealingManager re-enqueue tasks directly into our
|
||
// auto-merge queue when it clears a stale `merging` status, instead of
|
||
// relying on the 15s polling sweep to eventually catch them.
|
||
//
|
||
// Critically: clear the in-memory `mergeActive` entry before re-enqueueing.
|
||
// A stale-merge recovery means the prior merge attempt is dead — but its
|
||
// `try/finally` may never have fired (e.g. an AI provider call is wedged
|
||
// mid-await), so the entry is still in `mergeActive` and would otherwise
|
||
// cause `internalEnqueueMerge` to silently no-op.
|
||
//
|
||
// Tests substitute a minimal runtime mock that may not implement this hook.
|
||
this.runtime.setMergeEnqueuer?.((taskId) => {
|
||
// If the wedged attempt was the active one, abort its in-flight signal
|
||
// and dispose its session so subsequent code paths can release file
|
||
// handles / child processes promptly.
|
||
if (this.activeMergeTaskId === taskId) {
|
||
this.mergeAbortController?.abort();
|
||
this.mergeAbortController = null;
|
||
this.activeMergeSession?.dispose();
|
||
this.activeMergeSession = null;
|
||
this.activeMergeTaskId = null;
|
||
}
|
||
this.mergeActive.delete(taskId);
|
||
this.internalEnqueueMerge(taskId);
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Start the engine: initialize the runtime and all auxiliary subsystems.
|
||
*/
|
||
async start(): Promise<void> {
|
||
if (this.started) {
|
||
return;
|
||
}
|
||
|
||
// 1. Start the core runtime (TaskStore, Scheduler, Executor, Triage, etc.)
|
||
await this.runtime.start();
|
||
|
||
const store = this.runtime.getTaskStore();
|
||
const cwd = this.config.workingDirectory;
|
||
const settings = await store.getSettings();
|
||
|
||
if (typeof (store as { getResearchStore?: () => unknown }).getResearchStore === "function") {
|
||
this.researchOrchestrator = new ResearchOrchestrator({
|
||
store: store.getResearchStore(),
|
||
stepRunner: new ResearchStepRunner(),
|
||
maxConcurrentRuns: settings.researchMaxConcurrentRuns ?? 3,
|
||
});
|
||
}
|
||
|
||
this.remoteTunnelManager = new TunnelProcessManager();
|
||
try {
|
||
await this.restoreRemoteTunnelIfNeeded(store);
|
||
} catch (error) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
this.setRestoreDiagnostics("failed", "restore_start_failed", null, message);
|
||
runtimeLog.warn(`Remote tunnel restore evaluation failed (continuing startup): ${message}`);
|
||
}
|
||
|
||
// 2. Initialize PrMonitor + PrCommentHandler
|
||
this.prMonitor = new PrMonitor();
|
||
this.prCommentHandler = new PrCommentHandler(store);
|
||
this.prMonitor.onNewComments((taskId, prInfo, comments) =>
|
||
this.prCommentHandler!.handleNewComments(taskId, prInfo, comments),
|
||
);
|
||
this.runtime.configurePrMonitoring({
|
||
prMonitor: this.prMonitor,
|
||
onClosedPrFeedback: (taskId, prInfo, comments) =>
|
||
this.prCommentHandler!.createFollowUpTask(taskId, prInfo, comments),
|
||
});
|
||
|
||
// 3. Initialize notification services (unless caller manages them externally)
|
||
if (!this.options.skipNotifier) {
|
||
this.notificationService = new NotificationService(store, {
|
||
projectId: this.options.projectId,
|
||
ntfyBaseUrl: this.options.ntfyBaseUrl,
|
||
});
|
||
await this.notificationService.start();
|
||
|
||
// Backward-compatibility shim for gridlock notifications.
|
||
this.notifier = new NtfyNotifier(
|
||
store,
|
||
{
|
||
projectId: this.options.projectId,
|
||
ntfyBaseUrl: this.options.ntfyBaseUrl,
|
||
},
|
||
this.notificationService,
|
||
);
|
||
await this.notifier.start();
|
||
}
|
||
|
||
this.gridlockDetector = new GridlockDetector(store, {
|
||
onGridlock: (event) => this.notifier?.notifyGridlock(event),
|
||
onGridlockCleared: () => this.notifier?.notifyGridlock(null),
|
||
});
|
||
this.gridlockDetector.start();
|
||
|
||
// 4. Initialize AutomationStore + CronRunner
|
||
this.setAutomationSubsystemHealth(
|
||
"initializing",
|
||
"Initializing AutomationStore and CronRunner",
|
||
);
|
||
try {
|
||
const coreAutomationModule = await import("@fusion/core");
|
||
const { AutomationStore } = coreAutomationModule;
|
||
this.automationStore = new AutomationStore(cwd);
|
||
await this.automationStore.init();
|
||
|
||
const aiPromptExecutor = await createAiPromptExecutor(cwd);
|
||
this.cronRunner = new CronRunner(store, this.automationStore, {
|
||
aiPromptExecutor,
|
||
onScheduleRunProcessed: this.buildInsightRunHandler(cwd),
|
||
workingDirectory: cwd,
|
||
projectId: this.config.projectId,
|
||
scope: "project", // Project-scoped execution — global schedules run separately
|
||
});
|
||
|
||
const settings = await store.getSettings();
|
||
const startupSyncFailures: string[] = [];
|
||
|
||
// Sync insight extraction automation on startup
|
||
if (typeof coreAutomationModule.syncInsightExtractionAutomation === "function") {
|
||
try {
|
||
await coreAutomationModule.syncInsightExtractionAutomation(this.automationStore, settings);
|
||
} catch (err) {
|
||
const { message, detail } = formatErrorDetails(err);
|
||
startupSyncFailures.push(`insight extraction: ${message}`);
|
||
runtimeLog.warn(`Insight extraction automation startup sync failed:\n${detail}`);
|
||
}
|
||
} else {
|
||
runtimeLog.warn("syncInsightExtractionAutomation is unavailable; skipping startup sync");
|
||
}
|
||
|
||
// Sync auto-summarize automation on startup
|
||
if (typeof coreAutomationModule.syncAutoSummarizeAutomation === "function") {
|
||
try {
|
||
await coreAutomationModule.syncAutoSummarizeAutomation(this.automationStore, settings);
|
||
} catch (err) {
|
||
const { message, detail } = formatErrorDetails(err);
|
||
startupSyncFailures.push(`auto-summarize: ${message}`);
|
||
runtimeLog.warn(`Auto-summarize automation startup sync failed:\n${detail}`);
|
||
}
|
||
} else {
|
||
runtimeLog.warn("syncAutoSummarizeAutomation is unavailable; skipping startup sync");
|
||
}
|
||
|
||
// Sync memory dreams automation on startup
|
||
if (typeof coreAutomationModule.syncMemoryDreamsAutomation === "function") {
|
||
try {
|
||
await coreAutomationModule.syncMemoryDreamsAutomation(this.automationStore, settings);
|
||
} catch (err) {
|
||
const { message, detail } = formatErrorDetails(err);
|
||
startupSyncFailures.push(`memory dreams: ${message}`);
|
||
runtimeLog.warn(`Memory dreams automation startup sync failed:\n${detail}`);
|
||
}
|
||
} else {
|
||
runtimeLog.warn("syncMemoryDreamsAutomation is unavailable; skipping startup sync");
|
||
}
|
||
|
||
// Sync scheduled eval batch automation on startup
|
||
if (typeof coreAutomationModule.syncScheduledEvalBatchAutomation === "function") {
|
||
try {
|
||
await coreAutomationModule.syncScheduledEvalBatchAutomation(this.automationStore, settings);
|
||
} catch (err) {
|
||
const { message, detail } = formatErrorDetails(err);
|
||
startupSyncFailures.push(`scheduled eval: ${message}`);
|
||
runtimeLog.warn(`Scheduled eval automation startup sync failed:\n${detail}`);
|
||
}
|
||
} else {
|
||
runtimeLog.warn("syncScheduledEvalBatchAutomation is unavailable; skipping startup sync");
|
||
}
|
||
|
||
this.cronRunner.start();
|
||
|
||
if (startupSyncFailures.length > 0) {
|
||
this.setAutomationSubsystemHealth(
|
||
"degraded",
|
||
`CronRunner started with startup sync warnings: ${startupSyncFailures.join("; ")}`,
|
||
);
|
||
} else {
|
||
this.setAutomationSubsystemHealth(
|
||
"ready",
|
||
"CronRunner initialized and startup automation sync completed",
|
||
);
|
||
}
|
||
|
||
runtimeLog.log("CronRunner initialized and started");
|
||
} catch (err) {
|
||
// Non-fatal — automations are optional
|
||
const { message, detail } = formatErrorDetails(err);
|
||
this.cronRunner = undefined;
|
||
this.automationStore = undefined;
|
||
this.setAutomationSubsystemHealth(
|
||
"degraded",
|
||
`AutomationStore/CronRunner initialization failed: ${message}`,
|
||
);
|
||
runtimeLog.error(
|
||
`AutomationStore/CronRunner initialization failed (continuing without automations):\n${detail}`,
|
||
);
|
||
}
|
||
|
||
// 5. Wire settings event listeners
|
||
this.wireSettingsListeners(store);
|
||
|
||
// 6. Wire auto-merge on task:moved and task:updated pause interruptions
|
||
this.wireAutoMerge(store, cwd);
|
||
this.wireTaskPauseMergeInterruption(store);
|
||
|
||
// 7. Auto-merge startup sweep
|
||
await this.startupMergeSweep(store);
|
||
|
||
// 8. Start periodic merge retry sweep
|
||
this.scheduleMergeRetry(store);
|
||
|
||
this.started = true;
|
||
runtimeLog.log(`ProjectEngine started for ${this.config.projectId}`);
|
||
}
|
||
|
||
/**
|
||
* Gracefully stop the engine and all subsystems.
|
||
*
|
||
* If a merge is currently running, its abort signal is triggered before the
|
||
* active merge session is disposed so merge pipeline checkpoints can exit
|
||
* promptly without continuing git/verification work after shutdown starts.
|
||
*/
|
||
async stop(): Promise<void> {
|
||
if (!this.started) {
|
||
return;
|
||
}
|
||
|
||
this.shuttingDown = true;
|
||
|
||
// Stop merge retry timer
|
||
if (this.mergeRetryTimer) {
|
||
clearTimeout(this.mergeRetryTimer);
|
||
this.mergeRetryTimer = null;
|
||
}
|
||
|
||
// Abort active/pending merge work before tearing down sessions.
|
||
this.mergeAbortController?.abort();
|
||
this.mergeAbortController = null;
|
||
this.activeMergeTaskId = null;
|
||
this.pausedReviewTaskIds.clear();
|
||
|
||
const queuedTaskIds = [...this.mergeQueue];
|
||
this.mergeQueue.length = 0;
|
||
for (const queuedTaskId of queuedTaskIds) {
|
||
this.mergeActive.delete(queuedTaskId);
|
||
}
|
||
|
||
// Terminate active merge session
|
||
if (this.activeMergeSession) {
|
||
this.activeMergeSession.dispose();
|
||
this.activeMergeSession = null;
|
||
}
|
||
|
||
// Reject any pending manual merge promises
|
||
for (const [taskId, resolver] of this.manualMergeResolvers) {
|
||
resolver.reject(new Error(`Engine shutting down — merge for ${taskId} aborted`));
|
||
}
|
||
this.manualMergeResolvers.clear();
|
||
|
||
// Remove event listeners
|
||
try {
|
||
const store = this.runtime.getTaskStore();
|
||
for (const handler of this.settingsHandlers) {
|
||
store.off("settings:updated", handler);
|
||
}
|
||
if (this.taskMovedHandler) {
|
||
store.off("task:moved", this.taskMovedHandler);
|
||
}
|
||
if (this.taskUpdatedHandler) {
|
||
store.off("task:updated", this.taskUpdatedHandler);
|
||
}
|
||
} catch {
|
||
// Store may not be initialized if start() failed partway
|
||
}
|
||
|
||
// Stop auxiliary subsystems
|
||
this.notificationService?.stop();
|
||
this.notifier?.stop();
|
||
this.gridlockDetector?.stop();
|
||
this.cronRunner?.stop();
|
||
this.setAutomationSubsystemHealth("not-initialized", "Automation subsystem stopped");
|
||
|
||
const tunnelManager = this.remoteTunnelManager;
|
||
this.remoteTunnelManager = undefined;
|
||
if (tunnelManager) {
|
||
let shutdownStore: TaskStore | null = null;
|
||
try {
|
||
shutdownStore = this.runtime.getTaskStore();
|
||
} catch {
|
||
shutdownStore = null;
|
||
}
|
||
|
||
if (shutdownStore) {
|
||
try {
|
||
await this.persistShutdownRemoteLifecycle(shutdownStore, tunnelManager.getStatus());
|
||
} catch (error) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
runtimeLog.warn(`Failed to persist remote lifecycle shutdown markers: ${message}`);
|
||
}
|
||
}
|
||
|
||
try {
|
||
await tunnelManager.stop();
|
||
} catch (error) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
runtimeLog.warn(`Tunnel process manager stop failed (continuing shutdown): ${message}`);
|
||
}
|
||
}
|
||
|
||
// Stop the core runtime (Triage, Scheduler, Executor, etc.)
|
||
await this.runtime.stop();
|
||
|
||
this.started = false;
|
||
this.shuttingDown = false;
|
||
runtimeLog.log(`ProjectEngine stopped for ${this.config.projectId}`);
|
||
}
|
||
|
||
// ── Public accessors ──
|
||
|
||
/** Get the underlying InProcessRuntime. */
|
||
getRuntime(): InProcessRuntime {
|
||
return this.runtime;
|
||
}
|
||
|
||
/** Get the TaskStore. Throws if not started. */
|
||
getTaskStore(): TaskStore {
|
||
return this.runtime.getTaskStore();
|
||
}
|
||
|
||
/** Get the AgentStore (if initialized). Returns undefined before start(). */
|
||
getAgentStore(): import("@fusion/core").AgentStore | undefined {
|
||
return this.runtime.getAgentStore();
|
||
}
|
||
|
||
/** Get the MessageStore (if initialized). Returns undefined before start(). */
|
||
getMessageStore(): import("@fusion/core").MessageStore | undefined {
|
||
return this.runtime.getMessageStore();
|
||
}
|
||
|
||
/** Get the HeartbeatMonitor (if initialized). */
|
||
getHeartbeatMonitor() {
|
||
return this.runtime.getHeartbeatMonitor();
|
||
}
|
||
|
||
/** Get the project working directory. */
|
||
getWorkingDirectory(): string {
|
||
return this.config.workingDirectory;
|
||
}
|
||
|
||
/** Get the PrMonitor (if initialized). */
|
||
getPrMonitor(): PrMonitor | undefined {
|
||
return this.prMonitor;
|
||
}
|
||
|
||
/** Get the CronRunner (if initialized). */
|
||
getCronRunner(): CronRunner | undefined {
|
||
return this.cronRunner;
|
||
}
|
||
|
||
/** Get the AutomationStore (if initialized). */
|
||
getAutomationStore(): AutomationStoreType | undefined {
|
||
return this.automationStore;
|
||
}
|
||
|
||
/**
|
||
* Get the automation subsystem health for diagnostics and status reporting.
|
||
*/
|
||
getAutomationSubsystemHealth(): AutomationSubsystemHealth {
|
||
return { ...this.automationSubsystemHealth };
|
||
}
|
||
|
||
/** Get the RoutineStore (if initialized). */
|
||
getRoutineStore(): import("@fusion/core").RoutineStore | undefined {
|
||
return this.runtime.getRoutineStore();
|
||
}
|
||
|
||
/** Get the ResearchOrchestrator (if initialized). Returns undefined before start(). */
|
||
getResearchOrchestrator(): ResearchOrchestrator | undefined {
|
||
return this.researchOrchestrator;
|
||
}
|
||
|
||
/** Get the remote tunnel manager (available after start()). */
|
||
getRemoteTunnelManager(): TunnelProcessManager | undefined {
|
||
return this.remoteTunnelManager;
|
||
}
|
||
|
||
getRemoteTunnelRestoreDiagnostics(): TunnelRestoreDiagnostics {
|
||
return { ...this.remoteTunnelRestoreDiagnostics };
|
||
}
|
||
|
||
async startRemoteTunnel(): Promise<TunnelStatusSnapshot> {
|
||
const manager = this.remoteTunnelManager;
|
||
if (!manager) {
|
||
throw new Error("remote_tunnel_unavailable:remote tunnel manager is not initialized");
|
||
}
|
||
|
||
const store = this.runtime.getTaskStore();
|
||
const settings = await store.getSettings();
|
||
const remoteAccess = settings.remoteAccess;
|
||
if (!remoteAccess || !isRemoteActive(remoteAccess)) {
|
||
throw new Error("invalid_config:no remote access provider enabled");
|
||
}
|
||
|
||
const provider = remoteAccess.activeProvider;
|
||
if (!provider) {
|
||
throw new Error("invalid_config:no active remote provider configured");
|
||
}
|
||
|
||
const lifecycle = await this.evaluateRemoteLifecycle(settings, provider);
|
||
if (!lifecycle.config) {
|
||
throw new Error(`${lifecycle.reason ?? "invalid_config"}:${lifecycle.message ?? "remote provider prerequisites are not met"}`);
|
||
}
|
||
|
||
const current = manager.getStatus();
|
||
if (current.state === "running" && current.provider === provider) {
|
||
await this.writeRemoteLifecycleState(store, remoteAccess, {
|
||
...remoteAccess.lifecycle,
|
||
wasRunningOnShutdown: true,
|
||
lastRunningProvider: provider,
|
||
});
|
||
return manager.getStatus();
|
||
}
|
||
|
||
if (current.state === "running" && current.provider && current.provider !== provider) {
|
||
await manager.switchProvider(provider, lifecycle.config);
|
||
} else {
|
||
await manager.start(provider, lifecycle.config);
|
||
}
|
||
|
||
await this.writeRemoteLifecycleState(store, remoteAccess, {
|
||
...remoteAccess.lifecycle,
|
||
wasRunningOnShutdown: true,
|
||
lastRunningProvider: provider,
|
||
});
|
||
|
||
return manager.getStatus();
|
||
}
|
||
|
||
async stopRemoteTunnel(): Promise<TunnelStatusSnapshot> {
|
||
const manager = this.remoteTunnelManager;
|
||
if (!manager) {
|
||
throw new Error("remote_tunnel_unavailable:remote tunnel manager is not initialized");
|
||
}
|
||
|
||
await manager.stop();
|
||
|
||
const store = this.runtime.getTaskStore();
|
||
const settings = await store.getSettings();
|
||
const remoteAccess = settings.remoteAccess;
|
||
if (remoteAccess) {
|
||
await this.writeRemoteLifecycleState(store, remoteAccess, {
|
||
...remoteAccess.lifecycle,
|
||
wasRunningOnShutdown: false,
|
||
lastRunningProvider: null,
|
||
});
|
||
}
|
||
|
||
return manager.getStatus();
|
||
}
|
||
|
||
async detectExternalTunnel(): Promise<ExternalTunnelInfo | null> {
|
||
const manager = this.remoteTunnelManager;
|
||
if (!manager) {
|
||
return null;
|
||
}
|
||
|
||
const settings = await this.runtime.getTaskStore().getSettings();
|
||
const provider = settings.remoteAccess?.activeProvider ?? null;
|
||
if (provider !== "tailscale") {
|
||
return null;
|
||
}
|
||
|
||
return manager.detectExternalFunnel();
|
||
}
|
||
|
||
async killExternalTunnel(): Promise<void> {
|
||
const manager = this.remoteTunnelManager;
|
||
if (!manager) {
|
||
return;
|
||
}
|
||
|
||
const settings = await this.runtime.getTaskStore().getSettings();
|
||
const provider = settings.remoteAccess?.activeProvider ?? null;
|
||
if (provider !== "tailscale") {
|
||
return;
|
||
}
|
||
|
||
await manager.killExternalFunnel();
|
||
}
|
||
|
||
/** Get the RoutineRunner (if initialized). */
|
||
getRoutineRunner(): RoutineRunner | undefined {
|
||
return this.runtime.getRoutineRunner();
|
||
}
|
||
|
||
/** Get the HeartbeatTriggerScheduler from the underlying runtime, if initialized. */
|
||
getHeartbeatTriggerScheduler(): HeartbeatTriggerScheduler | undefined {
|
||
return this.runtime.getTriggerScheduler();
|
||
}
|
||
|
||
/**
|
||
* Enqueue a task ID for auto-merge if it is not already queued or active.
|
||
* Exposed publicly so callers can integrate the engine's merge queue with
|
||
* an external `onMerge` callback (e.g. dashboard's createServer call).
|
||
*/
|
||
enqueueMerge(taskId: string): void {
|
||
this.internalEnqueueMerge(taskId);
|
||
}
|
||
|
||
/**
|
||
* Perform an AI-powered merge for a task, serialized through the merge queue.
|
||
* This is the manual "merge now" path — it shares the same queue as auto-merge
|
||
* so only one merge runs at a time per project.
|
||
* Returns the full MergeResult so it can be used as the `onMerge` callback
|
||
* in createServer().
|
||
*/
|
||
async onMerge(taskId: string): Promise<MergeResult> {
|
||
// If this task is already queued or actively merging, wait for the
|
||
// existing merge to finish rather than starting a second one.
|
||
if (this.mergeActive.has(taskId)) {
|
||
return new Promise<MergeResult>((resolve, reject) => {
|
||
this.manualMergeResolvers.set(taskId, { resolve, reject });
|
||
// Don't re-enqueue — the task is already in the queue/active
|
||
});
|
||
}
|
||
|
||
return new Promise<MergeResult>((resolve, reject) => {
|
||
this.manualMergeResolvers.set(taskId, { resolve, reject });
|
||
this.internalEnqueueMerge(taskId);
|
||
});
|
||
}
|
||
|
||
private setRestoreDiagnostics(
|
||
outcome: TunnelRestoreDiagnostics["outcome"],
|
||
reason: TunnelRestoreReasonCode,
|
||
provider: TunnelProvider | null,
|
||
message?: string,
|
||
): void {
|
||
this.remoteTunnelRestoreDiagnostics = {
|
||
outcome,
|
||
reason,
|
||
provider,
|
||
message,
|
||
at: new Date().toISOString(),
|
||
};
|
||
}
|
||
|
||
private setAutomationSubsystemHealth(
|
||
status: AutomationSubsystemHealth["status"],
|
||
message: string,
|
||
): void {
|
||
this.automationSubsystemHealth = {
|
||
status,
|
||
message,
|
||
updatedAt: new Date().toISOString(),
|
||
};
|
||
}
|
||
|
||
private async restoreRemoteTunnelIfNeeded(store: TaskStore): Promise<void> {
|
||
const manager = this.remoteTunnelManager;
|
||
if (!manager) {
|
||
return;
|
||
}
|
||
|
||
const settings = await store.getSettings();
|
||
const remoteAccess = settings.remoteAccess;
|
||
if (!remoteAccess || !isRemoteActive(remoteAccess)) {
|
||
this.setRestoreDiagnostics("skipped", "remote_access_disabled", null);
|
||
return;
|
||
}
|
||
|
||
const lifecycle = remoteAccess.lifecycle;
|
||
if (!lifecycle.rememberLastRunning) {
|
||
this.setRestoreDiagnostics("skipped", "remember_last_running_disabled", null);
|
||
if (lifecycle.wasRunningOnShutdown || lifecycle.lastRunningProvider) {
|
||
await this.writeRemoteLifecycleState(store, remoteAccess, {
|
||
...lifecycle,
|
||
wasRunningOnShutdown: false,
|
||
lastRunningProvider: null,
|
||
});
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (!lifecycle.wasRunningOnShutdown) {
|
||
this.setRestoreDiagnostics("skipped", "no_prior_running_marker", null);
|
||
return;
|
||
}
|
||
|
||
const provider = lifecycle.lastRunningProvider ?? remoteAccess.activeProvider;
|
||
if (!provider) {
|
||
this.setRestoreDiagnostics("skipped", "provider_missing", null);
|
||
await this.writeRemoteLifecycleState(store, remoteAccess, {
|
||
...lifecycle,
|
||
wasRunningOnShutdown: false,
|
||
lastRunningProvider: null,
|
||
});
|
||
return;
|
||
}
|
||
|
||
const evaluation = await this.evaluateRemoteLifecycle(settings, provider);
|
||
if (!evaluation.config) {
|
||
this.setRestoreDiagnostics("skipped", evaluation.reason ?? "provider_not_configured", provider, evaluation.message);
|
||
await this.writeRemoteLifecycleState(store, remoteAccess, {
|
||
...lifecycle,
|
||
wasRunningOnShutdown: false,
|
||
lastRunningProvider: null,
|
||
});
|
||
return;
|
||
}
|
||
|
||
try {
|
||
await manager.start(provider, evaluation.config);
|
||
this.setRestoreDiagnostics("applied", "restore_started", provider);
|
||
await this.writeRemoteLifecycleState(store, remoteAccess, {
|
||
...lifecycle,
|
||
wasRunningOnShutdown: true,
|
||
lastRunningProvider: provider,
|
||
}, provider);
|
||
} catch (error) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
this.setRestoreDiagnostics("failed", "restore_start_failed", provider, message);
|
||
runtimeLog.warn(`Remote tunnel restore failed for ${provider}: ${message}`);
|
||
await this.writeRemoteLifecycleState(store, remoteAccess, {
|
||
...lifecycle,
|
||
wasRunningOnShutdown: false,
|
||
lastRunningProvider: null,
|
||
});
|
||
}
|
||
}
|
||
|
||
private async persistShutdownRemoteLifecycle(
|
||
store: TaskStore,
|
||
status: TunnelStatusSnapshot,
|
||
): Promise<void> {
|
||
const settings = await store.getSettings();
|
||
const remoteAccess = settings.remoteAccess;
|
||
if (!remoteAccess) {
|
||
return;
|
||
}
|
||
|
||
const shouldRememberRunning =
|
||
(status.state === "running" || status.state === "starting" || status.state === "stopping") &&
|
||
status.provider !== null;
|
||
|
||
await this.writeRemoteLifecycleState(store, remoteAccess, {
|
||
...remoteAccess.lifecycle,
|
||
wasRunningOnShutdown: shouldRememberRunning,
|
||
lastRunningProvider: shouldRememberRunning ? status.provider : null,
|
||
}, shouldRememberRunning ? status.provider : remoteAccess.activeProvider);
|
||
}
|
||
|
||
private async writeRemoteLifecycleState(
|
||
store: TaskStore,
|
||
remoteAccess: NonNullable<Settings["remoteAccess"]>,
|
||
lifecycle: NonNullable<Settings["remoteAccess"]>["lifecycle"],
|
||
activeProviderOverride?: TunnelProvider | null,
|
||
): Promise<void> {
|
||
await store.updateSettings({
|
||
remoteAccess: {
|
||
...remoteAccess,
|
||
activeProvider: activeProviderOverride === undefined ? remoteAccess.activeProvider : activeProviderOverride,
|
||
lifecycle,
|
||
},
|
||
});
|
||
}
|
||
|
||
private async evaluateRemoteLifecycle(
|
||
settings: Settings,
|
||
provider: TunnelProvider,
|
||
): Promise<RemoteLifecycleEvaluation> {
|
||
const remoteAccess = settings.remoteAccess;
|
||
if (!remoteAccess || !isRemoteActive(remoteAccess)) {
|
||
return { provider, reason: "remote_access_disabled", message: "No remote provider is enabled" };
|
||
}
|
||
|
||
if (provider === "tailscale") {
|
||
const tailscale = remoteAccess.providers.tailscale;
|
||
if (!tailscale.enabled) {
|
||
return { provider, reason: "provider_not_enabled", message: "Tailscale provider is disabled" };
|
||
}
|
||
if (!Number.isFinite(tailscale.targetPort) || tailscale.targetPort <= 0) {
|
||
return { provider, reason: "provider_not_configured", message: "Tailscale target port must be configured" };
|
||
}
|
||
|
||
const executable = await this.checkExecutableAvailable("tailscale");
|
||
if (!executable.available) {
|
||
return { provider, reason: "runtime_prerequisite_missing", message: executable.message };
|
||
}
|
||
|
||
return {
|
||
provider,
|
||
config: {
|
||
provider: "tailscale",
|
||
executablePath: "tailscale",
|
||
args: ["funnel", String(Math.floor(tailscale.targetPort))],
|
||
},
|
||
};
|
||
}
|
||
|
||
const cloudflare = remoteAccess.providers.cloudflare;
|
||
if (!cloudflare.enabled) {
|
||
return { provider, reason: "provider_not_enabled", message: "Cloudflare provider is disabled" };
|
||
}
|
||
if (cloudflare.quickTunnel === true) {
|
||
const executable = await this.checkExecutableAvailable("cloudflared");
|
||
if (!executable.available) {
|
||
return { provider, reason: "runtime_prerequisite_missing", message: executable.message };
|
||
}
|
||
|
||
return {
|
||
provider,
|
||
config: {
|
||
provider: "cloudflare",
|
||
quickTunnel: true,
|
||
executablePath: "cloudflared",
|
||
args: ["tunnel", "--url", "http://localhost:4040"],
|
||
},
|
||
};
|
||
}
|
||
|
||
if (!cloudflare.tunnelName?.trim() || !cloudflare.ingressUrl?.trim()) {
|
||
return { provider, reason: "provider_not_configured", message: "Cloudflare tunnel name and ingress URL must be configured" };
|
||
}
|
||
if (!cloudflare.tunnelToken?.trim()) {
|
||
return { provider, reason: "provider_not_configured", message: "Cloudflare tunnel token is required" };
|
||
}
|
||
|
||
const executable = await this.checkExecutableAvailable("cloudflared");
|
||
if (!executable.available) {
|
||
return { provider, reason: "runtime_prerequisite_missing", message: executable.message };
|
||
}
|
||
|
||
return {
|
||
provider,
|
||
config: {
|
||
provider: "cloudflare",
|
||
executablePath: "cloudflared",
|
||
args: ["tunnel", "--no-autoupdate", "run", cloudflare.tunnelName.trim()],
|
||
tokenEnvVar: "TUNNEL_TOKEN",
|
||
env: {
|
||
TUNNEL_TOKEN: cloudflare.tunnelToken,
|
||
},
|
||
},
|
||
};
|
||
}
|
||
|
||
private async checkExecutableAvailable(command: string): Promise<{ available: boolean; message?: string }> {
|
||
const checker = process.platform === "win32" ? "where" : "which";
|
||
try {
|
||
await execFileAsync(checker, [command]);
|
||
return { available: true };
|
||
} catch {
|
||
return {
|
||
available: false,
|
||
message: `${command} is not available on PATH`,
|
||
};
|
||
}
|
||
}
|
||
|
||
// ── Merge eligibility helpers (richer logic from dashboard.ts) ──
|
||
|
||
/**
|
||
* True when a retry-exhausted task in "in-review" has a verification buffer
|
||
* failure that can be auto-healed by resetting mergeRetries and re-running.
|
||
*/
|
||
private hasAutoHealableVerificationBufferFailure(task: {
|
||
mergeRetries?: number | null;
|
||
column: string;
|
||
error?: string | null;
|
||
log?: Array<{ action?: string }>;
|
||
}): boolean {
|
||
if (task.column !== "in-review") return false;
|
||
if ((task.mergeRetries ?? 0) < ProjectEngine.MAX_AUTO_MERGE_RETRIES) return false;
|
||
const err = task.error ?? "";
|
||
const matchesVerificationError =
|
||
err.includes("Deterministic test verification failed") ||
|
||
err.includes("Deterministic build verification failed") ||
|
||
err.includes("Build verification failed") ||
|
||
err.includes("Test verification failed");
|
||
if (!matchesVerificationError) return false;
|
||
|
||
return (
|
||
task.log?.some(
|
||
(entry) =>
|
||
entry.action?.includes("[verification] test command failed (exit 0)") ||
|
||
entry.action?.includes("[verification] build command failed (exit 0)") ||
|
||
entry.action?.includes("output exceeded buffer"),
|
||
) ?? false
|
||
);
|
||
}
|
||
|
||
/**
|
||
* True when a retry-exhausted task has been idle long enough for a
|
||
* 30-minute cooldown merge attempt.
|
||
*/
|
||
private isRetryCooldownElapsed(task: { updatedAt?: string | null }): boolean {
|
||
if (!task.updatedAt) return false;
|
||
const updated = Date.parse(task.updatedAt);
|
||
if (Number.isNaN(updated)) return false;
|
||
return Date.now() - updated >= ProjectEngine.AUTO_MERGE_COOLDOWN_MS;
|
||
}
|
||
|
||
/**
|
||
* Returns true if the task is eligible for auto-merge. Uses richer eligibility
|
||
* checks: merge blocker, retry limit, auto-heal patterns, cooldown elapsed.
|
||
*/
|
||
private canMergeTask(task: {
|
||
id?: string;
|
||
mergeRetries?: number | null;
|
||
column: string;
|
||
paused?: boolean;
|
||
status?: string | null;
|
||
error?: string | null;
|
||
steps?: Array<{ status: string }>;
|
||
workflowStepResults?: Array<{ status: string }>;
|
||
log?: Array<{ action?: string }>;
|
||
updatedAt?: string | null;
|
||
mergeDetails?: { mergeConfirmed?: boolean } | null;
|
||
}): boolean {
|
||
// Already-confirmed merges always eligible — just need to move to done
|
||
if (task.mergeDetails?.mergeConfirmed) return true;
|
||
if (this.options.getTaskMergeBlocker?.(task as Task)) return false;
|
||
// Terminal failure: don't let the cooldown sweep re-attempt a merge that
|
||
// already gave up (verification cap, conflict-bounce cap, or non-conflict
|
||
// error). The task is parked for human/follow-up intervention.
|
||
if (task.status === "failed") return false;
|
||
return (
|
||
(task.mergeRetries ?? 0) < ProjectEngine.MAX_AUTO_MERGE_RETRIES ||
|
||
this.hasAutoHealableVerificationBufferFailure(task) ||
|
||
this.isRetryCooldownElapsed(task)
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Remove and return the highest-priority taskId from the merge queue.
|
||
* Ordering: priority (urgent→low), then createdAt ASC, then id ASC — matching
|
||
* the triage and scheduler comparators. Manual merges (onMerge resolvers) are
|
||
* preferred over auto-merges so awaited callers aren't starved by a flood of
|
||
* higher-priority auto-enqueues. IDs whose tasks can't be loaded fall back to
|
||
* FIFO order so they still drain.
|
||
*/
|
||
private async pickNextMergeTaskId(store: TaskStore): Promise<string | undefined> {
|
||
if (this.mergeQueue.length === 0) return undefined;
|
||
// Fast path: with a single queued task there's nothing to reorder. Avoid an
|
||
// extra getTask round-trip (and keep callers that mock getTask once happy).
|
||
if (this.mergeQueue.length === 1) {
|
||
return this.mergeQueue.shift();
|
||
}
|
||
|
||
// Snapshot the queue before awaiting. While we await store.getTask for
|
||
// each id, stop() may clear mergeQueue and pause-handling may filter
|
||
// entries out — so we never trust positional indices afterwards.
|
||
const queueSnapshot = [...this.mergeQueue];
|
||
const entries: Array<{ taskId: string; task: Task | undefined; manual: boolean; order: number }> = [];
|
||
for (let i = 0; i < queueSnapshot.length; i++) {
|
||
const taskId = queueSnapshot[i]!;
|
||
const task = (await store.getTask(taskId).catch(() => undefined)) as Task | undefined;
|
||
entries.push({
|
||
taskId,
|
||
task,
|
||
manual: this.manualMergeResolvers.has(taskId),
|
||
order: i,
|
||
});
|
||
}
|
||
|
||
if (this.shuttingDown) return undefined;
|
||
|
||
entries.sort((a, b) => {
|
||
if (a.manual !== b.manual) return a.manual ? -1 : 1;
|
||
if (a.task && b.task) return compareTasksByPriorityThenAgeAndId(a.task, b.task);
|
||
if (a.task) return -1;
|
||
if (b.task) return 1;
|
||
return a.order - b.order;
|
||
});
|
||
|
||
// Find the highest-priority entry that is still in the live queue.
|
||
// Concurrent mutations (pause filter, stop) may have removed entries.
|
||
for (const entry of entries) {
|
||
const liveIndex = this.mergeQueue.indexOf(entry.taskId);
|
||
if (liveIndex !== -1) {
|
||
this.mergeQueue.splice(liveIndex, 1);
|
||
return entry.taskId;
|
||
}
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
private internalEnqueueMerge(taskId: string): void {
|
||
if (this.shuttingDown) return;
|
||
if (this.mergeActive.has(taskId)) return;
|
||
this.mergeActive.add(taskId);
|
||
this.mergeQueue.push(taskId);
|
||
void this.drainMergeQueue().catch((err: unknown) => {
|
||
runtimeLog.error(
|
||
`Merge queue drain failed unexpectedly: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Filter a sweep's listTasks() result to merge-eligible tasks, sort by
|
||
* priority (urgent → low, then createdAt ASC, then id ASC), and enqueue.
|
||
* Sorting before enqueue matters because each enqueue may immediately
|
||
* trigger drainMergeQueue's single-item fast path, so the first task
|
||
* pushed wins. listTasks returns createdAt ASC — without this sort an
|
||
* older low-priority task would start before a later urgent one.
|
||
*/
|
||
private enqueueEligibleInReviewTasks(tasks: readonly Task[]): number {
|
||
const eligible = sortTasksByPriorityThenAgeAndId(
|
||
tasks.filter((t) => !t.paused && this.canMergeTask(t as any)) as Task[],
|
||
);
|
||
for (const t of eligible) {
|
||
this.internalEnqueueMerge(t.id);
|
||
}
|
||
return eligible.length;
|
||
}
|
||
|
||
private async findActiveRecoveryFollowUp(
|
||
store: TaskStore,
|
||
parentTaskId: string,
|
||
branch?: string,
|
||
): Promise<{ task: Task; reason: "parent" | "branch" } | null> {
|
||
const tasks = await store.listTasks({ slim: true }).catch(() => [] as Task[]);
|
||
const activeRecoveryTasks = tasks.filter(
|
||
(task) =>
|
||
task.column !== "done" &&
|
||
task.column !== "archived" &&
|
||
task.sourceType === "recovery",
|
||
);
|
||
|
||
const sameParent = activeRecoveryTasks.find(
|
||
(task) => task.sourceParentTaskId === parentTaskId,
|
||
);
|
||
if (sameParent) return { task: sameParent, reason: "parent" };
|
||
|
||
if (branch) {
|
||
const sameBranch = activeRecoveryTasks.find((task) => task.branch === branch);
|
||
if (sameBranch) return { task: sameBranch, reason: "branch" };
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
private async drainMergeQueue(): Promise<void> {
|
||
if (this.mergeRunning) return;
|
||
this.mergeRunning = true;
|
||
|
||
try {
|
||
const store = this.runtime.getTaskStore();
|
||
const cwd = this.config.workingDirectory;
|
||
|
||
while (this.mergeQueue.length > 0 && !this.shuttingDown) {
|
||
const taskId = await this.pickNextMergeTaskId(store);
|
||
if (!taskId) break;
|
||
// pickNextMergeTaskId awaits store.getTask; re-check shutdown so we
|
||
// don't start a merge whose queue entry was cleared by stop().
|
||
if (this.shuttingDown) break;
|
||
const manualResolver = this.manualMergeResolvers.get(taskId);
|
||
try {
|
||
// Manual merges (onMerge) skip auto-merge eligibility checks
|
||
if (!manualResolver) {
|
||
// Re-check autoMerge and pause before each merge
|
||
const settings = await store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) {
|
||
runtimeLog.log(
|
||
`Auto-merge skipping ${taskId} — ${settings.globalPause ? "global pause" : "engine paused"} active`,
|
||
);
|
||
continue;
|
||
}
|
||
if (!settings.autoMerge) {
|
||
runtimeLog.log(`Auto-merge skipping ${taskId} — autoMerge disabled`);
|
||
continue;
|
||
}
|
||
|
||
const task = await store.getTask(taskId);
|
||
if (!task || task.column !== "in-review") {
|
||
continue;
|
||
}
|
||
if (task.paused) {
|
||
runtimeLog.log(`Auto-merge skipping ${taskId} — task is paused`);
|
||
continue;
|
||
}
|
||
|
||
// Intentional cast to access Task properties needed by merge validation
|
||
|
||
if (!this.canMergeTask(task as any)) {
|
||
continue;
|
||
}
|
||
|
||
// Fast path: merge already confirmed (e.g. task was moved back to
|
||
// in-review by auto-recovery after a successful merge) — just
|
||
// complete the task without re-running the merge process.
|
||
if (task.mergeDetails?.mergeConfirmed) {
|
||
runtimeLog.log(
|
||
`Auto-merge: ${taskId} already has mergeConfirmed — moving to done`,
|
||
);
|
||
await store.logEntry(
|
||
taskId,
|
||
"Merge already confirmed; completing task (recovered from post-merge state inconsistency)",
|
||
);
|
||
await store.updateTask(taskId, { status: null });
|
||
await store.moveTask(taskId, "done");
|
||
continue;
|
||
}
|
||
|
||
// Auto-heal verification buffer failures by resetting retry counter
|
||
|
||
if (this.hasAutoHealableVerificationBufferFailure(task as any)) {
|
||
await store.logEntry(
|
||
taskId,
|
||
"Auto-healing stale deterministic verification buffer failure; retrying merge verification",
|
||
);
|
||
await store.updateTask(taskId, { mergeRetries: 0, error: null, status: null });
|
||
} else if (
|
||
(task.mergeRetries ?? 0) >= ProjectEngine.MAX_AUTO_MERGE_RETRIES &&
|
||
|
||
this.isRetryCooldownElapsed(task as any)
|
||
) {
|
||
await store.logEntry(
|
||
taskId,
|
||
`Auto-merge retry cooldown elapsed (${Math.round(ProjectEngine.AUTO_MERGE_COOLDOWN_MS / 60000)}m idle); resetting retries for another attempt`,
|
||
);
|
||
await store.updateTask(taskId, { mergeRetries: 0 });
|
||
}
|
||
}
|
||
|
||
const settings = await store.getSettings();
|
||
|
||
// Cross-process guard: check if another process is already merging a
|
||
// task for this project. The in-memory mergeQueue serializes within
|
||
// this process, but multiple processes (e.g. dashboard + serve) share
|
||
// the same SQLite database and can race.
|
||
const activeMergingTask = store.getActiveMergingTask(taskId);
|
||
if (activeMergingTask) {
|
||
const retryMs = settings.pollIntervalMs ?? 15_000;
|
||
runtimeLog.log(
|
||
`Merge deferred for ${taskId} — ${activeMergingTask} is already merging (cross-process guard, retry in ${retryMs / 1000}s)`,
|
||
);
|
||
// Temporarily remove the manual resolver so the finally block
|
||
// doesn't prematurely resolve it. The re-enqueue will restore it.
|
||
if (manualResolver) {
|
||
this.manualMergeResolvers.delete(taskId);
|
||
}
|
||
// Re-queue after the poll interval so we retry once the other merge finishes
|
||
setTimeout(() => {
|
||
if (this.shuttingDown) {
|
||
manualResolver?.reject(new Error("Engine shutting down"));
|
||
return;
|
||
}
|
||
if (manualResolver) {
|
||
this.manualMergeResolvers.set(taskId, manualResolver);
|
||
}
|
||
this.internalEnqueueMerge(taskId);
|
||
}, retryMs);
|
||
continue;
|
||
}
|
||
|
||
const mergeStrategy = this.options.getMergeStrategy?.(settings) ?? "direct";
|
||
|
||
if (mergeStrategy === "pull-request" && this.options.processPullRequestMerge) {
|
||
this.activeMergeTaskId = taskId;
|
||
runtimeLog.log(`${manualResolver ? "Manual" : "Auto"}-merge processing PR flow for ${taskId}...`);
|
||
const result = await this.options.processPullRequestMerge(store, cwd, taskId);
|
||
if (result === "merged") {
|
||
runtimeLog.log(`${manualResolver ? "Manual" : "Auto"}-merge PR merged: ${taskId}`);
|
||
} else if (result === "waiting") {
|
||
runtimeLog.log(`${manualResolver ? "Manual" : "Auto"}-merge PR waiting: ${taskId}`);
|
||
}
|
||
if (manualResolver) {
|
||
// PR merge path doesn't produce a full MergeResult — fetch the task
|
||
// and construct one so the dashboard endpoint can respond.
|
||
const prTask = await store.getTask(taskId).catch(() => null);
|
||
this.manualMergeResolvers.delete(taskId);
|
||
manualResolver.resolve({
|
||
task: prTask!,
|
||
branch: prTask?.branch ?? "",
|
||
merged: result === "merged",
|
||
worktreeRemoved: false,
|
||
branchDeleted: false,
|
||
} as MergeResult);
|
||
}
|
||
} else {
|
||
// Direct merge via AI agent, gated by semaphore
|
||
runtimeLog.log(`${manualResolver ? "Manual" : "Auto"}-merge merging ${taskId}...`);
|
||
|
||
const semaphore = (this.runtime as any).globalSemaphore;
|
||
|
||
const pool = (this.runtime as any).worktreePool;
|
||
|
||
const agentStore = (this.runtime as any).agentStore;
|
||
|
||
const usageLimitPauser = (this.runtime as any).usageLimitPauser;
|
||
|
||
const rawMerge = () => {
|
||
this.activeMergeTaskId = taskId;
|
||
this.mergeAbortController = new AbortController();
|
||
return aiMergeTask(store, cwd, taskId, {
|
||
pool,
|
||
usageLimitPauser,
|
||
agentStore,
|
||
signal: this.mergeAbortController.signal,
|
||
onSession: (session) => {
|
||
this.activeMergeSession = session;
|
||
},
|
||
});
|
||
};
|
||
|
||
let result: MergeResult;
|
||
if (semaphore) {
|
||
result = await semaphore.run(rawMerge, PRIORITY_MERGE);
|
||
} else {
|
||
result = await rawMerge();
|
||
}
|
||
|
||
this.activeMergeSession = null;
|
||
runtimeLog.log(`${manualResolver ? "Manual" : "Auto"}-merge merged: ${taskId}`);
|
||
|
||
if (manualResolver) {
|
||
this.manualMergeResolvers.delete(taskId);
|
||
manualResolver.resolve(result);
|
||
}
|
||
|
||
// Reset retries on success
|
||
const latestTask = await store.getTask(taskId).catch(() => null);
|
||
if (latestTask?.mergeRetries && latestTask.mergeRetries > 0) {
|
||
await store.updateTask(taskId, { mergeRetries: 0 });
|
||
}
|
||
}
|
||
} catch (err: unknown) {
|
||
this.activeMergeSession = null;
|
||
const errorMsg = err instanceof Error ? err.message : String(err);
|
||
const mergeWasAborted = err instanceof Error && err.name === "MergeAbortedError";
|
||
|
||
if (mergeWasAborted) {
|
||
runtimeLog.log(`${manualResolver ? "Manual" : "Auto"}-merge aborted for ${taskId}: ${errorMsg}`);
|
||
this.mergeAbortController = null;
|
||
if (manualResolver) {
|
||
this.manualMergeResolvers.delete(taskId);
|
||
manualResolver.reject(err instanceof Error ? err : new Error(errorMsg));
|
||
} else {
|
||
await store.updateTask(taskId, { status: null }).catch(() => undefined);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
runtimeLog.error(`${manualResolver ? "Manual" : "Auto"}-merge failed for ${taskId}: ${errorMsg}`);
|
||
|
||
// Surface every merge failure on the task log so the dashboard shows
|
||
// *why* a merge didn't complete instead of silently looping.
|
||
await store
|
||
.logEntry(
|
||
taskId,
|
||
`${manualResolver ? "Manual" : "Auto"}-merge failed: ${errorMsg}`,
|
||
err instanceof Error ? err.name : undefined,
|
||
)
|
||
.catch((logErr: unknown) => {
|
||
runtimeLog.warn(
|
||
`Auto-merge: failed to log merge-failure entry on ${taskId}: ${logErr instanceof Error ? logErr.message : String(logErr)}`,
|
||
);
|
||
});
|
||
|
||
// If this was a manual merge, reject the promise and skip auto-retry logic
|
||
if (manualResolver) {
|
||
this.manualMergeResolvers.delete(taskId);
|
||
manualResolver.reject(err instanceof Error ? err : new Error(errorMsg));
|
||
continue;
|
||
}
|
||
|
||
const settingsOnErr = await store
|
||
.getSettings()
|
||
.catch(() => ({ autoResolveConflicts: true }));
|
||
const taskOnErr = await store.getTask(taskId).catch(() => null);
|
||
const mergeStrategyOnErr =
|
||
this.options.getMergeStrategy?.(settingsOnErr as Settings) ?? "direct";
|
||
|
||
// Deterministic verification failure: move back to in-progress
|
||
const isVerificationError =
|
||
err instanceof Error && err.name === "VerificationError" ||
|
||
errorMsg.includes("Deterministic test verification failed") ||
|
||
errorMsg.includes("Deterministic build verification failed");
|
||
|
||
if (taskOnErr && isVerificationError) {
|
||
const failedKind = errorMsg.includes("build verification") ? "build" : "test";
|
||
const previousBounces = taskOnErr.verificationFailureCount ?? 0;
|
||
const nextBounces = previousBounces + 1;
|
||
const cap = ProjectEngine.MAX_VERIFICATION_FAILURE_BOUNCES;
|
||
|
||
if (nextBounces >= cap) {
|
||
// Cap reached — stop bouncing the task and create a follow-up.
|
||
// The original task stays in in-review with status=failed so a
|
||
// human can inspect; the follow-up captures the failure context
|
||
// so a fresh agent can investigate (often a flaky test or an
|
||
// unrelated regression that won't be fixed by re-running this
|
||
// task's branch).
|
||
try {
|
||
await store.updateTask(taskId, {
|
||
status: "failed",
|
||
verificationFailureCount: nextBounces,
|
||
error: `Deterministic ${failedKind} verification failed ${nextBounces}× — auto-merge giving up to avoid infinite retry loop. See follow-up task for investigation.`,
|
||
});
|
||
const followUpDescription =
|
||
`Investigate repeated ${failedKind} verification failure on ${taskId} (${taskOnErr.title || "untitled"}). ` +
|
||
`Auto-merge attempted to fix and re-verify ${nextBounces} times without success — likely a flaky test or unrelated regression rather than a fix this task can produce on its own. ` +
|
||
`Look at the most recent [verification] log entries on ${taskId} for the failing command and output, then either fix the underlying issue or quarantine the flake.`;
|
||
const existingFollowUp = await this.findActiveRecoveryFollowUp(store, taskId);
|
||
if (existingFollowUp) {
|
||
await store.addTaskComment(
|
||
taskId,
|
||
`Auto-merge giving up after ${nextBounces} verification-failure bounces. Reusing existing follow-up ${existingFollowUp.task.id}.`,
|
||
"agent",
|
||
);
|
||
await store.logEntry(
|
||
taskId,
|
||
`Auto-merge gave up after ${nextBounces} verification-failure bounces — skipped creating duplicate follow-up (existing ${existingFollowUp.task.id})`,
|
||
"VerificationError",
|
||
);
|
||
runtimeLog.warn(
|
||
`Auto-merge: ${taskId} hit verification-failure cap (${nextBounces}/${cap}) — skipped duplicate follow-up (existing ${existingFollowUp.task.id})`,
|
||
);
|
||
} else {
|
||
const followUp = await store.createTask({
|
||
description: followUpDescription,
|
||
column: "triage",
|
||
priority: "high",
|
||
source: {
|
||
sourceType: "recovery",
|
||
sourceParentTaskId: taskId,
|
||
},
|
||
});
|
||
await store.addTaskComment(
|
||
taskId,
|
||
`Auto-merge giving up after ${nextBounces} verification-failure bounces. Created follow-up ${followUp.id} to investigate.`,
|
||
"agent",
|
||
);
|
||
await store.logEntry(
|
||
taskId,
|
||
`Auto-merge gave up after ${nextBounces} verification-failure bounces — created follow-up ${followUp.id}`,
|
||
"VerificationError",
|
||
);
|
||
runtimeLog.warn(
|
||
`Auto-merge: ${taskId} hit verification-failure cap (${nextBounces}/${cap}) — failed task and created follow-up ${followUp.id}`,
|
||
);
|
||
}
|
||
} catch (followUpErr) {
|
||
runtimeLog.error(
|
||
`Auto-merge: failed to fail-and-followup ${taskId} after verification cap: ${followUpErr instanceof Error ? followUpErr.message : String(followUpErr)}`,
|
||
);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
// Under cap — bounce back as before, but record the increment.
|
||
try {
|
||
await store.addTaskComment(
|
||
taskId,
|
||
`Deterministic ${failedKind} verification failed during merge (attempt ${nextBounces}/${cap}). ` +
|
||
`See the prior [verification] log entry for the truncated command output. ` +
|
||
`Please fix the failing ${failedKind} and push the update so the merge can retry.`,
|
||
"agent",
|
||
);
|
||
await store.updateTask(taskId, {
|
||
status: "merging-fix",
|
||
mergeRetries: 0,
|
||
error: null,
|
||
verificationFailureCount: nextBounces,
|
||
});
|
||
await store.moveTask(taskId, "in-progress");
|
||
await store.logEntry(
|
||
taskId,
|
||
`Deterministic ${failedKind} verification failed (${nextBounces}/${cap}) — moved back to in-progress with status=merging-fix for remediation`,
|
||
);
|
||
runtimeLog.log(
|
||
`Auto-merge: ${taskId} deterministic ${failedKind} verification failed (${nextBounces}/${cap}) — moved to in-progress with status=merging-fix`,
|
||
);
|
||
} catch {
|
||
runtimeLog.error(
|
||
`Auto-merge: failed to return ${taskId} to in-progress after verification failure`,
|
||
);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (mergeStrategyOnErr === "direct") {
|
||
const isConflictError =
|
||
errorMsg.includes("conflict") || errorMsg.includes("Conflict");
|
||
|
||
if (taskOnErr && isConflictError) {
|
||
const currentRetries = taskOnErr.mergeRetries ?? 0;
|
||
|
||
// Use `currentRetries + 1 < MAX` (not `currentRetries < MAX`) so
|
||
// the LAST retry's failure goes straight to the bounce code in
|
||
// this same engine tick. The previous condition scheduled a
|
||
// separate Nth setTimeout attempt — if the engine restarted
|
||
// before that timer fired (common during dev), the task was
|
||
// stranded with mergeRetries=MAX and only the cooldown sweep
|
||
// could ever try again (silent loop).
|
||
if (
|
||
(settingsOnErr as Settings).autoResolveConflicts !== false &&
|
||
currentRetries + 1 < ProjectEngine.MAX_AUTO_MERGE_RETRIES
|
||
) {
|
||
const newRetryCount = currentRetries + 1;
|
||
await store.updateTask(taskId, { mergeRetries: newRetryCount, status: null });
|
||
|
||
// Exponential backoff: 5s, 10s, 20s
|
||
const delayMs = 5000 * Math.pow(2, currentRetries);
|
||
runtimeLog.log(
|
||
`Auto-merge conflict retry ${newRetryCount}/${ProjectEngine.MAX_AUTO_MERGE_RETRIES} for ${taskId} in ${delayMs / 1000}s`,
|
||
);
|
||
setTimeout(() => {
|
||
if (!this.shuttingDown) this.internalEnqueueMerge(taskId);
|
||
}, delayMs);
|
||
} else {
|
||
// Conflict retries exhausted (or auto-resolve disabled).
|
||
// Previous behavior: silently clear status, leaving the task in
|
||
// in-review with mergeRetries=MAX. The 30-min cooldown sweep
|
||
// would then reset retries and re-attempt the same impossible
|
||
// merge forever, with no error surface for the user.
|
||
//
|
||
// New behavior: bounce the task back to in-progress so the
|
||
// executor can rebase against the latest main and retry. Cap
|
||
// bounces at MAX_MERGE_CONFLICT_BOUNCES — past that, park in
|
||
// in-review with status=failed and create a follow-up task so
|
||
// a human can resolve the conflict manually.
|
||
const previousBounces = taskOnErr.mergeConflictBounceCount ?? 0;
|
||
const nextBounces = previousBounces + 1;
|
||
const bounceCap = ProjectEngine.MAX_MERGE_CONFLICT_BOUNCES;
|
||
const autoResolveDisabled =
|
||
(settingsOnErr as Settings).autoResolveConflicts === false;
|
||
|
||
if (autoResolveDisabled || nextBounces > bounceCap) {
|
||
// Park for human intervention.
|
||
const reason = autoResolveDisabled
|
||
? "autoResolveConflicts is disabled"
|
||
: `merge-conflict bounce cap reached (${nextBounces - 1}/${bounceCap})`;
|
||
try {
|
||
await store.updateTask(taskId, {
|
||
status: "failed",
|
||
mergeRetries: ProjectEngine.MAX_AUTO_MERGE_RETRIES,
|
||
error: `Auto-merge gave up: ${reason}. ${errorMsg}`,
|
||
});
|
||
await store.addTaskComment(
|
||
taskId,
|
||
`Auto-merge gave up after ${ProjectEngine.MAX_AUTO_MERGE_RETRIES} conflict-resolution retries (${reason}). ` +
|
||
`Resolve the conflict on branch \`${taskOnErr.branch ?? "?"}\` manually, then unpause/retry.`,
|
||
"agent",
|
||
);
|
||
await store.logEntry(
|
||
taskId,
|
||
`Auto-merge gave up after conflict retries exhausted (${reason}); task parked for human intervention`,
|
||
"MergeConflictGiveUp",
|
||
);
|
||
if (!autoResolveDisabled) {
|
||
// Create a follow-up only when we capped on bounces; if
|
||
// auto-resolve is just disabled, the user is presumed to
|
||
// be handling merges manually and a follow-up is noise.
|
||
try {
|
||
const existingFollowUp = await this.findActiveRecoveryFollowUp(
|
||
store,
|
||
taskId,
|
||
taskOnErr.branch,
|
||
);
|
||
if (existingFollowUp) {
|
||
const dedupReason =
|
||
existingFollowUp.reason === "branch"
|
||
? `active recovery already owns branch \`${taskOnErr.branch ?? "?"}\``
|
||
: "active recovery already exists for this parent task";
|
||
await store.addTaskComment(
|
||
taskId,
|
||
`Auto-merge recovery follow-up already exists (${existingFollowUp.task.id}; ${dedupReason}). Skipping duplicate follow-up creation.`,
|
||
"agent",
|
||
);
|
||
await store.logEntry(
|
||
taskId,
|
||
`Auto-merge conflict recovery skipped duplicate follow-up (existing ${existingFollowUp.task.id}; ${dedupReason})`,
|
||
"MergeConflictGiveUp",
|
||
);
|
||
runtimeLog.warn(
|
||
`Auto-merge: ${taskId} conflict give-up skipped duplicate follow-up (existing ${existingFollowUp.task.id}; reason=${existingFollowUp.reason})`,
|
||
);
|
||
} else {
|
||
const followUp = await store.createTask({
|
||
description:
|
||
`Resolve auto-merge conflict on ${taskId} (${taskOnErr.title || "untitled"}). ` +
|
||
`Auto-merge attempted to rebase + resolve ${nextBounces - 1} times against main and exhausted retries each pass. ` +
|
||
`Branch: \`${taskOnErr.branch ?? "?"}\`. Worktree: \`${taskOnErr.worktree ?? "?"}\`. ` +
|
||
`Last merge error: ${errorMsg}`,
|
||
column: "triage",
|
||
priority: "high",
|
||
source: {
|
||
sourceType: "recovery",
|
||
sourceParentTaskId: taskId,
|
||
},
|
||
});
|
||
await store.addTaskComment(
|
||
taskId,
|
||
`Created follow-up ${followUp.id} to track manual conflict resolution.`,
|
||
"agent",
|
||
);
|
||
}
|
||
} catch (followUpErr) {
|
||
runtimeLog.warn(
|
||
`Auto-merge: failed to create follow-up for ${taskId}: ${followUpErr instanceof Error ? followUpErr.message : String(followUpErr)}`,
|
||
);
|
||
}
|
||
}
|
||
} catch (recoveryErr) {
|
||
runtimeLog.error(
|
||
`Auto-merge: failed to park ${taskId} after conflict-bounce cap: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)}`,
|
||
);
|
||
}
|
||
} else {
|
||
// Bounce to in-progress for a fresh rebase + retry pass.
|
||
try {
|
||
await store.addTaskComment(
|
||
taskId,
|
||
`Auto-merge could not resolve conflicts within ${ProjectEngine.MAX_AUTO_MERGE_RETRIES} retries (bounce ${nextBounces}/${bounceCap}). ` +
|
||
`Bouncing back to in-progress for a fresh rebase against main; the executor will re-run quality gates and re-attempt the merge.`,
|
||
"agent",
|
||
);
|
||
await store.updateTask(taskId, {
|
||
status: null,
|
||
mergeRetries: 0,
|
||
error: null,
|
||
mergeConflictBounceCount: nextBounces,
|
||
});
|
||
await store.moveTask(taskId, "in-progress");
|
||
await store.logEntry(
|
||
taskId,
|
||
`Auto-merge conflicts unresolved (${ProjectEngine.MAX_AUTO_MERGE_RETRIES}/${ProjectEngine.MAX_AUTO_MERGE_RETRIES}) — bounced to in-progress for re-rebase (bounce ${nextBounces}/${bounceCap})`,
|
||
"MergeConflictBounce",
|
||
);
|
||
runtimeLog.log(
|
||
`Auto-merge: ${taskId} conflict retries exhausted — bounced to in-progress (${nextBounces}/${bounceCap})`,
|
||
);
|
||
} catch (recoveryErr) {
|
||
runtimeLog.error(
|
||
`Auto-merge: failed to bounce ${taskId} after conflict exhaustion: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)}`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
// Non-conflict error — stop retrying until user intervenes.
|
||
// Mark status=failed so the cooldown sweep won't silently
|
||
// re-attempt; the catch-block-top logEntry already recorded the
|
||
// failure on the task log.
|
||
try {
|
||
await store.updateTask(taskId, {
|
||
status: "failed",
|
||
mergeRetries: ProjectEngine.MAX_AUTO_MERGE_RETRIES,
|
||
error: errorMsg,
|
||
});
|
||
await store.addTaskComment(
|
||
taskId,
|
||
`Auto-merge failed with a non-conflict error and stopped retrying: ${errorMsg}`,
|
||
"agent",
|
||
);
|
||
} catch (recoveryErr) {
|
||
runtimeLog.error(
|
||
`Auto-merge: failed to update ${taskId} after non-conflict error: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)}`,
|
||
);
|
||
}
|
||
}
|
||
} else {
|
||
// Non-direct merge strategy (e.g. pull-request) errored — park as
|
||
// failed so the cooldown sweep stops re-attempting silently.
|
||
try {
|
||
await store.updateTask(taskId, {
|
||
status: "failed",
|
||
mergeRetries: ProjectEngine.MAX_AUTO_MERGE_RETRIES,
|
||
error: errorMsg,
|
||
});
|
||
} catch (recoveryErr) {
|
||
runtimeLog.error(
|
||
`Auto-merge: failed to update ${taskId} after merge strategy error: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)}`,
|
||
);
|
||
}
|
||
}
|
||
} finally {
|
||
if (this.activeMergeTaskId === taskId) {
|
||
this.activeMergeTaskId = null;
|
||
}
|
||
this.mergeAbortController = null;
|
||
this.mergeActive.delete(taskId);
|
||
// If a manual merge was requested while this task was already in-flight,
|
||
// the resolver was set but not consumed above. Resolve it now.
|
||
const lateResolver = this.manualMergeResolvers.get(taskId);
|
||
if (lateResolver) {
|
||
this.manualMergeResolvers.delete(taskId);
|
||
const finalTask = await store.getTask(taskId).catch(() => null);
|
||
lateResolver.resolve({
|
||
task: finalTask!,
|
||
branch: finalTask?.branch ?? "",
|
||
merged: finalTask?.column === "done",
|
||
worktreeRemoved: false,
|
||
branchDeleted: false,
|
||
} as MergeResult);
|
||
}
|
||
}
|
||
}
|
||
} finally {
|
||
this.mergeRunning = false;
|
||
}
|
||
}
|
||
|
||
private wireAutoMerge(store: TaskStore, _cwd: string): void {
|
||
this.taskMovedHandler = async ({ task, to }: { task: Task; to: string }) => {
|
||
if (to !== "in-review") return;
|
||
if (task.paused) return;
|
||
if (this.options.getTaskMergeBlocker?.(task)) return;
|
||
|
||
// Grace period before handing off to the merger. The executor's finally
|
||
// block (session disposal, child-agent termination, in-flight reviewer
|
||
// teardown) runs *after* the moveTask("in-review") that fires this
|
||
// event. Without a delay, the merger's session can start emitting logs
|
||
// while the executor is still cleaning up — observed in FN-2910 as
|
||
// overlapping [reviewer]/[merger] log streams. The delay is also a
|
||
// belt-and-braces guard against any in-flight reviewer that the
|
||
// executor spawned just before transitioning.
|
||
setTimeout(async () => {
|
||
try {
|
||
// Re-validate eligibility after the grace period — the task may
|
||
// have been paused, moved, or had its merge blocked.
|
||
const latestTask = await store.getTask(task.id).catch(() => null);
|
||
if (!latestTask) return;
|
||
if (latestTask.column !== "in-review") return;
|
||
if (latestTask.paused) return;
|
||
if (this.options.getTaskMergeBlocker?.(latestTask)) return;
|
||
const settings = await store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return;
|
||
if (!settings.autoMerge) return;
|
||
this.internalEnqueueMerge(task.id);
|
||
} catch (err: unknown) {
|
||
runtimeLog.warn(
|
||
`Auto-merge: failed to read settings for task:moved on ${task.id}: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
}
|
||
}, MERGE_HANDOFF_GRACE_MS);
|
||
};
|
||
store.on("task:moved", this.taskMovedHandler);
|
||
}
|
||
|
||
private wireTaskPauseMergeInterruption(store: TaskStore): void {
|
||
this.taskUpdatedHandler = async (task: Task) => {
|
||
if (task.column !== "in-review") {
|
||
this.pausedReviewTaskIds.delete(task.id);
|
||
return;
|
||
}
|
||
|
||
if (task.paused) {
|
||
this.pausedReviewTaskIds.add(task.id);
|
||
|
||
const queueLengthBefore = this.mergeQueue.length;
|
||
this.mergeQueue = this.mergeQueue.filter((queuedTaskId) => queuedTaskId !== task.id);
|
||
const removedFromQueue = this.mergeQueue.length !== queueLengthBefore;
|
||
|
||
if (removedFromQueue) {
|
||
this.mergeActive.delete(task.id);
|
||
runtimeLog.log(`Paused in-review task removed from merge queue: ${task.id}`);
|
||
}
|
||
|
||
if (this.activeMergeTaskId !== task.id) {
|
||
return;
|
||
}
|
||
|
||
runtimeLog.log(`Paused in-review task interrupting active merge: ${task.id}`);
|
||
this.mergeAbortController?.abort();
|
||
this.mergeAbortController = null;
|
||
|
||
if (this.activeMergeSession) {
|
||
this.activeMergeSession.dispose();
|
||
this.activeMergeSession = null;
|
||
}
|
||
|
||
this.mergeActive.delete(task.id);
|
||
return;
|
||
}
|
||
|
||
const wasPaused = this.pausedReviewTaskIds.delete(task.id);
|
||
if (!wasPaused) {
|
||
return;
|
||
}
|
||
|
||
try {
|
||
const settings = await store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused || !settings.autoMerge) {
|
||
return;
|
||
}
|
||
if (this.options.getTaskMergeBlocker?.(task)) {
|
||
return;
|
||
}
|
||
|
||
runtimeLog.log(`Unpaused in-review task re-enqueued for auto-merge: ${task.id}`);
|
||
this.internalEnqueueMerge(task.id);
|
||
} catch (err: unknown) {
|
||
runtimeLog.warn(
|
||
`In-review unpause: failed to re-enqueue ${task.id} for auto-merge: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
}
|
||
};
|
||
|
||
store.on("task:updated", this.taskUpdatedHandler);
|
||
}
|
||
|
||
private async startupMergeSweep(store: TaskStore): Promise<void> {
|
||
try {
|
||
const tasks = await store.listTasks({ column: "in-review" });
|
||
|
||
// Clear stale "merging"/"merging-pr" statuses left by a prior crash.
|
||
// No merge is actually running at startup, so any task still marked
|
||
// as merging is a leftover from a previous engine lifecycle.
|
||
// This runs unconditionally (regardless of autoMerge setting) because
|
||
// stale statuses block manual merges too.
|
||
const staleStatuses = new Set(["merging", "merging-pr"]);
|
||
for (const t of tasks) {
|
||
if (t.status && staleStatuses.has(t.status)) {
|
||
runtimeLog.log(`Startup sweep: clearing stale '${t.status}' status on ${t.id}`);
|
||
await store.updateTask(t.id, { status: null });
|
||
// Update in-memory object so canMergeTask sees the cleared status
|
||
|
||
(t as any).status = null;
|
||
}
|
||
}
|
||
|
||
const settings = await store.getSettings();
|
||
if (!settings.autoMerge) return;
|
||
|
||
|
||
const enqueued = this.enqueueEligibleInReviewTasks(tasks as Task[]);
|
||
if (enqueued > 0) {
|
||
runtimeLog.log(`Auto-merge startup sweep: enqueueing ${enqueued} task(s)`);
|
||
}
|
||
} catch (err: unknown) {
|
||
runtimeLog.warn(
|
||
`Auto-merge startup sweep failed: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
private scheduleMergeRetry(store: TaskStore): void {
|
||
if (this.shuttingDown) return;
|
||
|
||
const schedule = async () => {
|
||
if (this.shuttingDown) return;
|
||
|
||
try {
|
||
const settings = await store.getSettings();
|
||
if (!settings.globalPause && !settings.enginePaused && settings.autoMerge) {
|
||
const tasks = await store.listTasks({ column: "in-review" });
|
||
this.enqueueEligibleInReviewTasks(tasks as Task[]);
|
||
}
|
||
} catch (err: unknown) {
|
||
runtimeLog.warn(
|
||
`Auto-merge periodic sweep failed: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
} finally {
|
||
if (!this.shuttingDown) {
|
||
let interval = 15_000;
|
||
try {
|
||
const settings = await store.getSettings();
|
||
interval = settings.pollIntervalMs ?? 15_000;
|
||
} catch (err: unknown) {
|
||
runtimeLog.warn(
|
||
`Auto-merge retry: failed to read pollIntervalMs, using default 15s: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
}
|
||
this.mergeRetryTimer = setTimeout(() => void schedule(), interval);
|
||
}
|
||
}
|
||
};
|
||
|
||
// Kick off the first sweep after a delay
|
||
this.mergeRetryTimer = setTimeout(() => void schedule(), 15_000);
|
||
}
|
||
|
||
// ── Settings event listeners ──
|
||
|
||
private async resumeAfterUnpauseAndSweepInReview(
|
||
store: TaskStore,
|
||
settings: Settings,
|
||
source: "Global unpause" | "Engine unpause",
|
||
): Promise<void> {
|
||
try {
|
||
const runtime = this.runtime as any;
|
||
runtime.resumeAfterUnpause?.().catch((err: Error) =>
|
||
runtimeLog.error(
|
||
`Failed to resume agentic activity on ${source.toLowerCase()}:`,
|
||
err,
|
||
),
|
||
);
|
||
} catch (err: unknown) {
|
||
runtimeLog.warn(
|
||
`${source}: failed to dispatch resumeAfterUnpause: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
}
|
||
|
||
if (settings.globalPause || settings.enginePaused || !settings.autoMerge) {
|
||
return;
|
||
}
|
||
|
||
try {
|
||
const tasks = await store.listTasks({ column: "in-review" });
|
||
this.enqueueEligibleInReviewTasks(tasks as Task[]);
|
||
} catch (err: unknown) {
|
||
runtimeLog.warn(
|
||
`${source}: failed to scan in-review tasks for auto-merge: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
private wireSettingsListeners(store: TaskStore): void {
|
||
const applyDetectorPauseLifecycle = (paused: boolean, source: string): void => {
|
||
try {
|
||
const detector = (this.runtime as any).stuckTaskDetector;
|
||
if (paused) {
|
||
detector?.pause?.();
|
||
} else {
|
||
detector?.resume?.();
|
||
}
|
||
} catch (err: unknown) {
|
||
runtimeLog.warn(
|
||
`${source}: stuck detector ${paused ? "pause" : "resume"} hook failed: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
}
|
||
};
|
||
|
||
// 1. Unified pause lifecycle — detector only resumes once BOTH pause sources
|
||
// are clear, and pauses when either source engages.
|
||
const onPauseLifecycleTransition = ({
|
||
settings: s,
|
||
previous: prev,
|
||
}: {
|
||
settings: Settings;
|
||
previous: Settings;
|
||
}) => {
|
||
const wasPaused = prev.globalPause || prev.enginePaused;
|
||
const isPaused = s.globalPause || s.enginePaused;
|
||
|
||
if (!wasPaused && isPaused) {
|
||
const source = s.globalPause && !prev.globalPause ? "Global pause" : "Engine pause";
|
||
applyDetectorPauseLifecycle(true, source);
|
||
}
|
||
|
||
if (wasPaused && !isPaused) {
|
||
const source = prev.globalPause && !s.globalPause ? "Global unpause" : "Engine unpause";
|
||
applyDetectorPauseLifecycle(false, source);
|
||
}
|
||
};
|
||
store.on("settings:updated", onPauseLifecycleTransition);
|
||
this.settingsHandlers.push(onPauseLifecycleTransition);
|
||
|
||
// 2. Global pause — terminate active merge session AND abort any running
|
||
// deterministic verification (pnpm test/build). The abort controller gates
|
||
// both the AI merge agent and the spawned child processes; without it,
|
||
// verification commands keep churning until they finish naturally.
|
||
const onGlobalPause = ({ settings, previous }: { settings: Settings; previous: Settings }) => {
|
||
if (settings.globalPause && !previous.globalPause) {
|
||
if (this.mergeAbortController) {
|
||
runtimeLog.log("Global pause — aborting in-flight merge verification");
|
||
this.mergeAbortController.abort();
|
||
this.mergeAbortController = null;
|
||
}
|
||
if (this.activeMergeSession) {
|
||
runtimeLog.log("Global pause — terminating active merge session");
|
||
this.activeMergeSession.dispose();
|
||
this.activeMergeSession = null;
|
||
}
|
||
}
|
||
};
|
||
store.on("settings:updated", onGlobalPause);
|
||
this.settingsHandlers.push(onGlobalPause);
|
||
|
||
// 3. Global unpause — resume orphaned tasks + sweep in-review
|
||
const onGlobalUnpause = async ({
|
||
settings: s,
|
||
previous: prev,
|
||
}: {
|
||
settings: Settings;
|
||
previous: Settings;
|
||
}) => {
|
||
if (prev.globalPause && !s.globalPause) {
|
||
runtimeLog.log("Global unpause — resuming agentic activity");
|
||
await this.resumeAfterUnpauseAndSweepInReview(store, s, "Global unpause");
|
||
}
|
||
};
|
||
store.on("settings:updated", onGlobalUnpause);
|
||
this.settingsHandlers.push(onGlobalUnpause);
|
||
|
||
// 4. Engine unpause — same as global unpause
|
||
const onEngineUnpause = async ({
|
||
settings: s,
|
||
previous: prev,
|
||
}: {
|
||
settings: Settings;
|
||
previous: Settings;
|
||
}) => {
|
||
if (prev.enginePaused && !s.enginePaused) {
|
||
runtimeLog.log("Engine unpaused — resuming agentic activity");
|
||
await this.resumeAfterUnpauseAndSweepInReview(store, s, "Engine unpause");
|
||
}
|
||
};
|
||
store.on("settings:updated", onEngineUnpause);
|
||
this.settingsHandlers.push(onEngineUnpause);
|
||
|
||
// 5. Stuck task timeout change — trigger immediate check
|
||
const onStuckTimeoutChange = async ({
|
||
settings: s,
|
||
previous: prev,
|
||
}: {
|
||
settings: Settings;
|
||
previous: Settings;
|
||
}) => {
|
||
if (s.taskStuckTimeoutMs !== prev.taskStuckTimeoutMs) {
|
||
runtimeLog.log(
|
||
`Stuck task timeout changed to ${s.taskStuckTimeoutMs}ms — running immediate check`,
|
||
);
|
||
try {
|
||
|
||
const detector = (this.runtime as any).stuckTaskDetector;
|
||
await detector?.checkNow?.();
|
||
} catch (err: unknown) {
|
||
runtimeLog.warn(
|
||
`Stuck-timeout change: detector.checkNow() failed: ${err instanceof Error ? err.message : String(err)}`,
|
||
);
|
||
}
|
||
}
|
||
};
|
||
store.on("settings:updated", onStuckTimeoutChange);
|
||
this.settingsHandlers.push(onStuckTimeoutChange);
|
||
|
||
// 5. Memory maintenance settings change — sync automations
|
||
const onInsightSettingsChange = async ({
|
||
settings: s,
|
||
previous: prev,
|
||
}: {
|
||
settings: Settings;
|
||
previous: Settings;
|
||
}) => {
|
||
const insightKeys = [
|
||
"insightExtractionEnabled",
|
||
"insightExtractionSchedule",
|
||
"insightExtractionMinIntervalMs",
|
||
] as const;
|
||
const dreamKeys = [
|
||
"memoryDreamsEnabled",
|
||
"memoryDreamsSchedule",
|
||
] as const;
|
||
|
||
|
||
const changed = insightKeys.some((key) => (s as any)[key] !== (prev as any)[key]);
|
||
|
||
const dreamsChanged = dreamKeys.some((key) => (s as any)[key] !== (prev as any)[key]);
|
||
if ((!changed && !dreamsChanged) || !this.automationStore) return;
|
||
|
||
try {
|
||
const { syncInsightExtractionAutomation, syncMemoryDreamsAutomation } = await import("@fusion/core");
|
||
if (changed && typeof syncInsightExtractionAutomation === "function") {
|
||
await syncInsightExtractionAutomation(this.automationStore, s);
|
||
runtimeLog.log("Insight extraction automation synced with settings");
|
||
}
|
||
if (dreamsChanged && typeof syncMemoryDreamsAutomation === "function") {
|
||
await syncMemoryDreamsAutomation(this.automationStore, s);
|
||
runtimeLog.log("Memory dreams automation synced with settings");
|
||
}
|
||
} catch (err) {
|
||
const { message, detail } = formatErrorDetails(err);
|
||
this.setAutomationSubsystemHealth(
|
||
"degraded",
|
||
`Failed to sync memory maintenance automation: ${message}`,
|
||
);
|
||
runtimeLog.warn(`Failed to sync memory maintenance automation:\n${detail}`);
|
||
}
|
||
};
|
||
store.on("settings:updated", onInsightSettingsChange);
|
||
this.settingsHandlers.push(onInsightSettingsChange);
|
||
|
||
// 6. Auto-summarize settings change — sync automation
|
||
const onAutoSummarizeSettingsChange = async ({
|
||
settings: s,
|
||
previous: prev,
|
||
}: {
|
||
settings: Settings;
|
||
previous: Settings;
|
||
}) => {
|
||
const autoSummarizeKeys = [
|
||
"memoryAutoSummarizeEnabled",
|
||
"memoryAutoSummarizeThresholdChars",
|
||
"memoryAutoSummarizeSchedule",
|
||
] as const;
|
||
|
||
|
||
const changed = autoSummarizeKeys.some((key) => (s as any)[key] !== (prev as any)[key]);
|
||
if (!changed || !this.automationStore) return;
|
||
|
||
try {
|
||
const { syncAutoSummarizeAutomation } = await import("@fusion/core");
|
||
if (typeof syncAutoSummarizeAutomation === "function") {
|
||
await syncAutoSummarizeAutomation(this.automationStore, s);
|
||
runtimeLog.log("Auto-summarize automation synced with settings");
|
||
}
|
||
} catch (err) {
|
||
const { message, detail } = formatErrorDetails(err);
|
||
this.setAutomationSubsystemHealth(
|
||
"degraded",
|
||
`Failed to sync auto-summarize automation: ${message}`,
|
||
);
|
||
runtimeLog.warn(`Failed to sync auto-summarize automation:\n${detail}`);
|
||
}
|
||
};
|
||
store.on("settings:updated", onAutoSummarizeSettingsChange);
|
||
this.settingsHandlers.push(onAutoSummarizeSettingsChange);
|
||
|
||
// 7. Scheduled eval settings change — sync automation
|
||
const onScheduledEvalSettingsChange = async ({
|
||
settings: s,
|
||
previous: prev,
|
||
}: {
|
||
settings: Settings;
|
||
previous: Settings;
|
||
}) => {
|
||
const evalKeys = [
|
||
"taskEvaluationEnabled",
|
||
"taskEvaluationSchedule",
|
||
] as const;
|
||
|
||
const changed = evalKeys.some((key) => (s as any)[key] !== (prev as any)[key]);
|
||
if (!changed || !this.automationStore) return;
|
||
|
||
try {
|
||
const { syncScheduledEvalBatchAutomation } = await import("@fusion/core");
|
||
if (typeof syncScheduledEvalBatchAutomation === "function") {
|
||
await syncScheduledEvalBatchAutomation(this.automationStore, s);
|
||
runtimeLog.log("Scheduled eval automation synced with settings");
|
||
}
|
||
} catch (err) {
|
||
const { message, detail } = formatErrorDetails(err);
|
||
this.setAutomationSubsystemHealth(
|
||
"degraded",
|
||
`Failed to sync scheduled eval automation: ${message}`,
|
||
);
|
||
runtimeLog.warn(`Failed to sync scheduled eval automation:\n${detail}`);
|
||
}
|
||
};
|
||
store.on("settings:updated", onScheduledEvalSettingsChange);
|
||
this.settingsHandlers.push(onScheduledEvalSettingsChange);
|
||
}
|
||
|
||
/**
|
||
* Build the onScheduleRunProcessed callback for CronRunner.
|
||
* Chains the built-in processAndAuditInsightExtraction with any
|
||
* caller-provided onInsightRunProcessed callback.
|
||
*/
|
||
private buildInsightRunHandler(
|
||
cwd: string,
|
||
): (schedule: ScheduledTask, result: AutomationRunResult) => Promise<void> {
|
||
const callerCallback = this.options.onInsightRunProcessed;
|
||
|
||
return async (schedule: ScheduledTask, result: AutomationRunResult): Promise<void> => {
|
||
// Invoke caller-provided callback first (e.g. for test hooks)
|
||
if (callerCallback) {
|
||
try {
|
||
await callerCallback(schedule, result);
|
||
} catch (err) {
|
||
runtimeLog.warn(
|
||
"onInsightRunProcessed callback error:",
|
||
err instanceof Error ? err.message : err,
|
||
);
|
||
}
|
||
}
|
||
|
||
// Run built-in processAndAuditInsightExtraction
|
||
try {
|
||
const { INSIGHT_EXTRACTION_SCHEDULE_NAME, processAndAuditInsightExtraction } =
|
||
await import("@fusion/core");
|
||
|
||
if (
|
||
typeof INSIGHT_EXTRACTION_SCHEDULE_NAME !== "string" ||
|
||
typeof processAndAuditInsightExtraction !== "function"
|
||
) {
|
||
return;
|
||
}
|
||
|
||
if (schedule.name !== INSIGHT_EXTRACTION_SCHEDULE_NAME) {
|
||
return;
|
||
}
|
||
|
||
const stepResults = result.stepResults ?? [];
|
||
const aiStep = stepResults.find(
|
||
(sr) =>
|
||
sr.stepName === "Extract Memory Insights and Prune" ||
|
||
sr.stepName === "Extract Memory Insights",
|
||
);
|
||
|
||
if (!aiStep) {
|
||
runtimeLog.log(`No insight extraction step found in ${schedule.name} result`);
|
||
return;
|
||
}
|
||
|
||
runtimeLog.log("Processing memory insight extraction run...");
|
||
|
||
const auditReport = await processAndAuditInsightExtraction(cwd, {
|
||
rawResponse: aiStep.output ?? "",
|
||
stepSuccess: aiStep.success,
|
||
runAt: result.startedAt,
|
||
error: aiStep.error,
|
||
});
|
||
|
||
const pruneStatus = auditReport.pruning.applied
|
||
? ` | Pruned: ${auditReport.pruning.originalSize} -> ${auditReport.pruning.newSize} chars`
|
||
: ` | Pruning: ${auditReport.pruning.reason}`;
|
||
|
||
runtimeLog.log(
|
||
`Memory audit complete — Health: ${auditReport.health}, ` +
|
||
`Insights: ${auditReport.insightsMemory.insightCount}${pruneStatus}`,
|
||
);
|
||
} catch (err) {
|
||
runtimeLog.warn(
|
||
"Failed to process insight extraction:",
|
||
err instanceof Error ? err.message : err,
|
||
);
|
||
}
|
||
};
|
||
}
|
||
}
|